diff --git a/accumulo-handler/pom.xml b/accumulo-handler/pom.xml index b00fe88..f14683a 100644 --- a/accumulo-handler/pom.xml +++ b/accumulo-handler/pom.xml @@ -19,7 +19,7 @@ org.apache.hive hive - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../pom.xml diff --git a/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/columns/ColumnMapper.java b/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/columns/ColumnMapper.java index ff9db46..0498bab 100644 --- a/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/columns/ColumnMapper.java +++ b/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/columns/ColumnMapper.java @@ -20,11 +20,11 @@ import java.util.Collections; import java.util.List; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.accumulo.AccumuloHiveConstants; import org.apache.hadoop.hive.accumulo.serde.TooManyAccumuloColumnsException; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.util.StringUtils; import org.apache.log4j.Logger; import com.google.common.base.Preconditions; diff --git a/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/columns/TestColumnMapper.java b/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/columns/TestColumnMapper.java index e5c1e61..e5f1b97 100644 --- a/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/columns/TestColumnMapper.java +++ b/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/columns/TestColumnMapper.java @@ -20,12 +20,12 @@ import java.util.Iterator; import java.util.List; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.accumulo.AccumuloHiveConstants; import org.apache.hadoop.hive.accumulo.serde.TooManyAccumuloColumnsException; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; -import org.apache.hadoop.util.StringUtils; import org.junit.Assert; import org.junit.Test; diff --git a/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/FirstCharAccumuloCompositeRowId.java b/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/FirstCharAccumuloCompositeRowId.java index e047ae5..8c1ec8d 100644 --- a/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/FirstCharAccumuloCompositeRowId.java +++ b/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/FirstCharAccumuloCompositeRowId.java @@ -19,10 +19,10 @@ import java.util.Arrays; import java.util.Properties; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef; import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazySimpleStructObjectInspector; -import org.apache.hadoop.util.StringUtils; import org.apache.log4j.Logger; /** diff --git a/ant/pom.xml b/ant/pom.xml index ff8a32a..c5caa24 100644 --- a/ant/pom.xml +++ b/ant/pom.xml @@ -19,7 +19,7 @@ org.apache.hive hive - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../pom.xml diff --git a/beeline/pom.xml b/beeline/pom.xml index 9a1c5dc..dcedf74 100644 --- a/beeline/pom.xml +++ b/beeline/pom.xml @@ -19,7 +19,7 @@ org.apache.hive hive - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../pom.xml diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLine.java b/beeline/src/java/org/apache/hive/beeline/BeeLine.java index e3c2449..8539a41 100644 --- a/beeline/src/java/org/apache/hive/beeline/BeeLine.java +++ b/beeline/src/java/org/apache/hive/beeline/BeeLine.java @@ -145,9 +145,11 @@ private final Map formats = map(new Object[] { "vertical", new VerticalOutputFormat(this), "table", new TableOutputFormat(this), - "csv", new SeparatedValuesOutputFormat(this, ','), - "tsv", new SeparatedValuesOutputFormat(this, '\t'), + "csv2", new SeparatedValuesOutputFormat(this, ','), + "tsv2", new SeparatedValuesOutputFormat(this, '\t'), "dsv", new SeparatedValuesOutputFormat(this, BeeLineOpts.DEFAULT_DELIMITER_FOR_DSV), + "csv", new DeprecatedSeparatedValuesOutputFormat(this, ','), + "tsv", new DeprecatedSeparatedValuesOutputFormat(this, '\t'), "xmlattr", new XMLAttributeOutputFormat(this), "xmlelements", new XMLElementOutputFormat(this), }); diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java b/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java index 2a0e9b5..f73fb44 100644 --- a/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java +++ b/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java @@ -427,6 +427,9 @@ public boolean getAutosave() { } public void setOutputFormat(String outputFormat) { + if(outputFormat.equalsIgnoreCase("csv") || outputFormat.equalsIgnoreCase("tsv")) { + beeLine.info("Format " + outputFormat + " is deprecated, please use " + outputFormat + "2"); + } this.outputFormat = outputFormat; } diff --git a/beeline/src/java/org/apache/hive/beeline/DeprecatedSeparatedValuesOutputFormat.java b/beeline/src/java/org/apache/hive/beeline/DeprecatedSeparatedValuesOutputFormat.java new file mode 100644 index 0000000..f49daee --- /dev/null +++ b/beeline/src/java/org/apache/hive/beeline/DeprecatedSeparatedValuesOutputFormat.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * This source file is based on code taken from SQLLine 1.0.2 + * See SQLLine notice in LICENSE + */ +package org.apache.hive.beeline; + + +/** + * OutputFormat for values separated by a delimiter. + * + * Note this does not handle escaping of the quote char. + * The new SeparatedValuesOutputFormat supports that. The formats supported by + * this class are deprecated. + * + */ +class DeprecatedSeparatedValuesOutputFormat implements OutputFormat { + + private final BeeLine beeLine; + private char separator; + + public DeprecatedSeparatedValuesOutputFormat(BeeLine beeLine, char separator) { + this.beeLine = beeLine; + setSeparator(separator); + } + + @Override + public int print(Rows rows) { + int count = 0; + while (rows.hasNext()) { + printRow(rows, (Rows.Row) rows.next()); + count++; + } + return count - 1; // sans header row + } + + public void printRow(Rows rows, Rows.Row row) { + String[] vals = row.values; + StringBuilder buf = new StringBuilder(); + for (int i = 0; i < vals.length; i++) { + buf.append(buf.length() == 0 ? "" : "" + getSeparator()) + .append('\'') + .append(vals[i] == null ? "" : vals[i]) + .append('\''); + } + beeLine.output(buf.toString()); + } + + public void setSeparator(char separator) { + this.separator = separator; + } + + public char getSeparator() { + return this.separator; + } +} diff --git a/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java b/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java index a21fa65..d13d8b6 100644 --- a/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java +++ b/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java @@ -17,7 +17,19 @@ */ package org.apache.hive.beeline; +import com.google.common.collect.Lists; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.HiveMetaException; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; import java.util.IllegalFormatException; +import java.util.List; public class HiveSchemaHelper { public static final String DB_DERBY = "derby"; @@ -26,6 +38,56 @@ public static final String DB_POSTGRACE = "postgres"; public static final String DB_ORACLE = "oracle"; + /*** + * Get JDBC connection to metastore db + * + * @param userName metastore connection username + * @param password metastore connection password + * @param printInfo print connection parameters + * @param hiveConf hive config object + * @return metastore connection object + * @throws org.apache.hadoop.hive.metastore.api.MetaException + */ + public static Connection getConnectionToMetastore(String userName, + String password, boolean printInfo, HiveConf hiveConf) + throws HiveMetaException { + try { + String connectionURL = getValidConfVar( + HiveConf.ConfVars.METASTORECONNECTURLKEY, hiveConf); + String driver = getValidConfVar( + HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER, hiveConf); + if (printInfo) { + System.out.println("Metastore connection URL:\t " + connectionURL); + System.out.println("Metastore Connection Driver :\t " + driver); + System.out.println("Metastore connection User:\t " + userName); + } + if ((userName == null) || userName.isEmpty()) { + throw new HiveMetaException("UserName empty "); + } + + // load required JDBC driver + Class.forName(driver); + + // Connect using the JDBC URL and user/pass from conf + return DriverManager.getConnection(connectionURL, userName, password); + } catch (IOException e) { + throw new HiveMetaException("Failed to get schema version.", e); + } catch (SQLException e) { + throw new HiveMetaException("Failed to get schema version.", e); + } catch (ClassNotFoundException e) { + throw new HiveMetaException("Failed to load driver", e); + } + } + + public static String getValidConfVar(HiveConf.ConfVars confVar, HiveConf hiveConf) + throws IOException { + String confVarStr = hiveConf.get(confVar.varname); + if (confVarStr == null || confVarStr.isEmpty()) { + throw new IOException("Empty " + confVar.varname); + } + return confVarStr; + } + public interface NestedScriptParser { public enum CommandType { @@ -57,7 +119,7 @@ public boolean isNestedScript(String dbCommand); /*** - * Find if the given command is should be passed to DB + * Find if the given command should not be passed to DB * @param dbCommand * @return */ @@ -80,8 +142,16 @@ * @return */ public boolean needsQuotedIdentifier(); - } + /*** + * Flatten the nested upgrade script into a buffer + * @param scriptDir upgrade script directory + * @param scriptFile upgrade script file + * @return string of sql commands + */ + public String buildCommand(String scriptDir, String scriptFile) + throws IllegalFormatException, IOException; + } /*** * Base implemenation of NestedScriptParser @@ -89,6 +159,18 @@ * */ private static abstract class AbstractCommandParser implements NestedScriptParser { + private List dbOpts; + private String msUsername; + private String msPassword; + private HiveConf hiveConf; + + public AbstractCommandParser(String dbOpts, String msUsername, String msPassword, + HiveConf hiveConf) { + setDbOpts(dbOpts); + this.msUsername = msUsername; + this.msPassword = msPassword; + this.hiveConf = hiveConf; + } @Override public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException{ @@ -127,13 +209,84 @@ public String cleanseCommand(String dbCommand) { public boolean needsQuotedIdentifier() { return false; } - } + @Override + public String buildCommand( + String scriptDir, String scriptFile) throws IllegalFormatException, IOException { + BufferedReader bfReader = + new BufferedReader(new FileReader(scriptDir + File.separatorChar + scriptFile)); + String currLine; + StringBuilder sb = new StringBuilder(); + String currentCommand = null; + while ((currLine = bfReader.readLine()) != null) { + currLine = currLine.trim(); + if (currLine.isEmpty()) { + continue; // skip empty lines + } + + if (currentCommand == null) { + currentCommand = currLine; + } else { + currentCommand = currentCommand + " " + currLine; + } + if (isPartialCommand(currLine)) { + // if its a partial line, continue collecting the pieces + continue; + } + + // if this is a valid executable command then add it to the buffer + if (!isNonExecCommand(currentCommand)) { + currentCommand = cleanseCommand(currentCommand); + + if (isNestedScript(currentCommand)) { + // if this is a nested sql script then flatten it + String currScript = getScriptName(currentCommand); + sb.append(buildCommand(scriptDir, currScript)); + } else { + // Now we have a complete statement, process it + // write the line to buffer + sb.append(currentCommand); + sb.append(System.getProperty("line.separator")); + } + } + currentCommand = null; + } + bfReader.close(); + return sb.toString(); + } + + private void setDbOpts(String dbOpts) { + if (dbOpts != null) { + this.dbOpts = Lists.newArrayList(dbOpts.split(",")); + } + } + + protected List getDbOpts() { + return dbOpts; + } + + protected String getMsUsername() { + return msUsername; + } + + protected String getMsPassword() { + return msPassword; + } + + protected HiveConf getHiveConf() { + return hiveConf; + } + } // Derby commandline parser public static class DerbyCommandParser extends AbstractCommandParser { private static String DERBY_NESTING_TOKEN = "RUN"; + public DerbyCommandParser(String dbOpts, String msUsername, String msPassword, + HiveConf hiveConf) { + super(dbOpts, msUsername, msPassword, hiveConf); + } + @Override public String getScriptName(String dbCommand) throws IllegalArgumentException { @@ -154,13 +307,17 @@ public boolean isNestedScript(String dbCommand) { } } - // MySQL parser public static class MySqlCommandParser extends AbstractCommandParser { private static final String MYSQL_NESTING_TOKEN = "SOURCE"; private static final String DELIMITER_TOKEN = "DELIMITER"; private String delimiter = DEFAUTL_DELIMITER; + public MySqlCommandParser(String dbOpts, String msUsername, String msPassword, + HiveConf hiveConf) { + super(dbOpts, msUsername, msPassword, hiveConf); + } + @Override public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException{ boolean isPartial = super.isPartialCommand(dbCommand); @@ -213,6 +370,11 @@ public String cleanseCommand(String dbCommand) { public static class PostgresCommandParser extends AbstractCommandParser { private static String POSTGRES_NESTING_TOKEN = "\\i"; + public PostgresCommandParser(String dbOpts, String msUsername, String msPassword, + HiveConf hiveConf) { + super(dbOpts, msUsername, msPassword, hiveConf); + } + @Override public String getScriptName(String dbCommand) throws IllegalArgumentException { String[] tokens = dbCommand.split(" "); @@ -237,6 +399,12 @@ public boolean needsQuotedIdentifier() { //Oracle specific parser public static class OracleCommandParser extends AbstractCommandParser { private static String ORACLE_NESTING_TOKEN = "@"; + + public OracleCommandParser(String dbOpts, String msUsername, String msPassword, + HiveConf hiveConf) { + super(dbOpts, msUsername, msPassword, hiveConf); + } + @Override public String getScriptName(String dbCommand) throws IllegalArgumentException { if (!isNestedScript(dbCommand)) { @@ -255,6 +423,12 @@ public boolean isNestedScript(String dbCommand) { //MSSQL specific parser public static class MSSQLCommandParser extends AbstractCommandParser { private static String MSSQL_NESTING_TOKEN = ":r"; + + public MSSQLCommandParser(String dbOpts, String msUsername, String msPassword, + HiveConf hiveConf) { + super(dbOpts, msUsername, msPassword, hiveConf); + } + @Override public String getScriptName(String dbCommand) throws IllegalArgumentException { String[] tokens = dbCommand.split(" "); @@ -271,18 +445,24 @@ public boolean isNestedScript(String dbCommand) { } public static NestedScriptParser getDbCommandParser(String dbName) { + return getDbCommandParser(dbName, null, null, null, null); + } + + public static NestedScriptParser getDbCommandParser(String dbName, + String dbOpts, String msUsername, String msPassword, + HiveConf hiveConf) { if (dbName.equalsIgnoreCase(DB_DERBY)) { - return new DerbyCommandParser(); + return new DerbyCommandParser(dbOpts, msUsername, msPassword, hiveConf); } else if (dbName.equalsIgnoreCase(DB_MSSQL)) { - return new MSSQLCommandParser(); + return new MSSQLCommandParser(dbOpts, msUsername, msPassword, hiveConf); } else if (dbName.equalsIgnoreCase(DB_MYSQL)) { - return new MySqlCommandParser(); + return new MySqlCommandParser(dbOpts, msUsername, msPassword, hiveConf); } else if (dbName.equalsIgnoreCase(DB_POSTGRACE)) { - return new PostgresCommandParser(); + return new PostgresCommandParser(dbOpts, msUsername, msPassword, hiveConf); } else if (dbName.equalsIgnoreCase(DB_ORACLE)) { - return new OracleCommandParser(); + return new OracleCommandParser(dbOpts, msUsername, msPassword, hiveConf); } else { throw new IllegalArgumentException("Unknown dbType " + dbName); } } -} +} \ No newline at end of file diff --git a/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java b/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java index c376687..8edf413 100644 --- a/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java +++ b/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java @@ -17,20 +17,16 @@ */ package org.apache.hive.beeline; -import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; -import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; import java.io.PrintStream; import java.sql.Connection; -import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; -import java.util.IllegalFormatException; import java.util.List; import org.apache.commons.cli.CommandLine; @@ -56,6 +52,7 @@ private String passWord = null; private boolean dryRun = false; private boolean verbose = false; + private String dbOpts = null; private final HiveConf hiveConf; private final String dbType; private final MetaStoreSchemaInfo metaStoreSchemaInfo; @@ -65,7 +62,7 @@ public HiveSchemaTool(String dbType) throws HiveMetaException { } public HiveSchemaTool(String hiveHome, HiveConf hiveConf, String dbType) - throws HiveMetaException { + throws HiveMetaException { if (hiveHome == null || hiveHome.isEmpty()) { throw new HiveMetaException("No Hive home directory provided"); } @@ -101,12 +98,27 @@ public void setVerbose(boolean verbose) { this.verbose = verbose; } + public void setDbOpts(String dbOpts) { + this.dbOpts = dbOpts; + } + private static void printAndExit(Options cmdLineOptions) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("schemaTool", cmdLineOptions); System.exit(1); } + private Connection getConnectionToMetastore(boolean printInfo) + throws HiveMetaException { + return HiveSchemaHelper.getConnectionToMetastore(userName, + passWord, printInfo, hiveConf); + } + + private NestedScriptParser getDbCommandParser(String dbType) { + return HiveSchemaHelper.getDbCommandParser(dbType, dbOpts, userName, + passWord, hiveConf); + } + /*** * Print Hive version and schema version * @throws MetaException @@ -121,9 +133,9 @@ public void showInfo() throws HiveMetaException { // read schema version from metastore private String getMetaStoreSchemaVersion(Connection metastoreConn) - throws HiveMetaException { + throws HiveMetaException { String versionQuery; - if (HiveSchemaHelper.getDbCommandParser(dbType).needsQuotedIdentifier()) { + if (getDbCommandParser(dbType).needsQuotedIdentifier()) { versionQuery = "select t.\"SCHEMA_VERSION\" from \"VERSION\" t"; } else { versionQuery = "select t.SCHEMA_VERSION from VERSION t"; @@ -152,40 +164,6 @@ private void testConnectionToMetastore() throws HiveMetaException { } } - /*** - * get JDBC connection to metastore db - * - * @param printInfo print connection parameters - * @return - * @throws MetaException - */ - private Connection getConnectionToMetastore(boolean printInfo) - throws HiveMetaException { - try { - String connectionURL = getValidConfVar(ConfVars.METASTORECONNECTURLKEY); - String driver = getValidConfVar(ConfVars.METASTORE_CONNECTION_DRIVER); - if (printInfo) { - System.out.println("Metastore connection URL:\t " + connectionURL); - System.out.println("Metastore Connection Driver :\t " + driver); - System.out.println("Metastore connection User:\t " + userName); - } - if ((userName == null) || userName.isEmpty()) { - throw new HiveMetaException("UserName empty "); - } - - // load required JDBC driver - Class.forName(driver); - - // Connect using the JDBC URL and user/pass from conf - return DriverManager.getConnection(connectionURL, userName, passWord); - } catch (IOException e) { - throw new HiveMetaException("Failed to get schema version.", e); - } catch (SQLException e) { - throw new HiveMetaException("Failed to get schema version.", e); - } catch (ClassNotFoundException e) { - throw new HiveMetaException("Failed to load driver", e); - } - } /** * check if the current schema version in metastore matches the Hive version @@ -196,8 +174,8 @@ public void verifySchemaVersion() throws HiveMetaException { if (dryRun) { return; } - String newSchemaVersion = - getMetaStoreSchemaVersion(getConnectionToMetastore(false)); + String newSchemaVersion = getMetaStoreSchemaVersion( + getConnectionToMetastore(false)); // verify that the new version is added to schema if (!MetaStoreSchemaInfo.getHiveSchemaVersion().equalsIgnoreCase(newSchemaVersion)) { throw new HiveMetaException("Found unexpected schema version " + newSchemaVersion); @@ -209,7 +187,8 @@ public void verifySchemaVersion() throws HiveMetaException { * @throws MetaException */ public void doUpgrade() throws HiveMetaException { - String fromVersion = getMetaStoreSchemaVersion(getConnectionToMetastore(false)); + String fromVersion = getMetaStoreSchemaVersion( + getConnectionToMetastore(false)); if (fromVersion == null || fromVersion.isEmpty()) { throw new HiveMetaException("Schema version not stored in the metastore. " + "Metastore schema is too old or corrupt. Try specifying the version manually"); @@ -240,6 +219,7 @@ public void doUpgrade(String fromSchemaVer) throws HiveMetaException { for (String scriptFile : upgradeScripts) { System.out.println("Upgrade script " + scriptFile); if (!dryRun) { + runPreUpgrade(scriptDir, scriptFile); runBeeLine(scriptDir, scriptFile); System.out.println("Completed " + scriptFile); } @@ -291,58 +271,47 @@ public void doInit(String toVersion) throws HiveMetaException { } } - // Flatten the nested upgrade script into a buffer - public static String buildCommand(NestedScriptParser dbCommandParser, - String scriptDir, String scriptFile) throws IllegalFormatException, IOException { - - BufferedReader bfReader = - new BufferedReader(new FileReader(scriptDir + File.separatorChar + scriptFile)); - String currLine; - StringBuilder sb = new StringBuilder(); - String currentCommand = null; - while ((currLine = bfReader.readLine()) != null) { - currLine = currLine.trim(); - if (currLine.isEmpty()) { - continue; // skip empty lines - } - - if (currentCommand == null) { - currentCommand = currLine; - } else { - currentCommand = currentCommand + " " + currLine; - } - if (dbCommandParser.isPartialCommand(currLine)) { - // if its a partial line, continue collecting the pieces - continue; + /** + * Run pre-upgrade scripts corresponding to a given upgrade script, + * if any exist. The errors from pre-upgrade are ignored. + * Pre-upgrade scripts typically contain setup statements which + * may fail on some database versions and failure is ignorable. + * + * @param scriptDir upgrade script directory name + * @param scriptFile upgrade script file name + */ + private void runPreUpgrade(String scriptDir, String scriptFile) { + for (int i = 0;; i++) { + String preUpgradeScript = + MetaStoreSchemaInfo.getPreUpgradeScriptName(i, scriptFile); + File preUpgradeScriptFile = new File(scriptDir, preUpgradeScript); + if (!preUpgradeScriptFile.isFile()) { + break; } - // if this is a valid executable command then add it to the buffer - if (!dbCommandParser.isNonExecCommand(currentCommand)) { - currentCommand = dbCommandParser.cleanseCommand(currentCommand); - - if (dbCommandParser.isNestedScript(currentCommand)) { - // if this is a nested sql script then flatten it - String currScript = dbCommandParser.getScriptName(currentCommand); - sb.append(buildCommand(dbCommandParser, scriptDir, currScript)); - } else { - // Now we have a complete statement, process it - // write the line to buffer - sb.append(currentCommand); - sb.append(System.getProperty("line.separator")); + try { + runBeeLine(scriptDir, preUpgradeScript); + System.out.println("Completed " + preUpgradeScript); + } catch (Exception e) { + // Ignore the pre-upgrade script errors + System.err.println("Warning in pre-upgrade script " + preUpgradeScript + ": " + + e.getMessage()); + if (verbose) { + e.printStackTrace(); } } - currentCommand = null; } - bfReader.close(); - return sb.toString(); } - // run beeline on the given metastore scrip, flatten the nested scripts into single file - private void runBeeLine(String scriptDir, String scriptFile) throws IOException { - NestedScriptParser dbCommandParser = - HiveSchemaHelper.getDbCommandParser(dbType); + /*** + * Run beeline with the given metastore script. Flatten the nested scripts + * into single file. + */ + private void runBeeLine(String scriptDir, String scriptFile) + throws IOException, HiveMetaException { + NestedScriptParser dbCommandParser = getDbCommandParser(dbType); // expand the nested script - String sqlCommands = buildCommand(dbCommandParser, scriptDir, scriptFile); + String sqlCommands = dbCommandParser.buildCommand(scriptDir, scriptFile); File tmpFile = File.createTempFile("schematool", ".sql"); tmpFile.deleteOnExit(); @@ -360,9 +329,11 @@ private void runBeeLine(String scriptDir, String scriptFile) throws IOException public void runBeeLine(String sqlScriptFile) throws IOException { List argList = new ArrayList(); argList.add("-u"); - argList.add(getValidConfVar(ConfVars.METASTORECONNECTURLKEY)); + argList.add(HiveSchemaHelper.getValidConfVar( + ConfVars.METASTORECONNECTURLKEY, hiveConf)); argList.add("-d"); - argList.add(getValidConfVar(ConfVars.METASTORE_CONNECTION_DRIVER)); + argList.add(HiveSchemaHelper.getValidConfVar( + ConfVars.METASTORE_CONNECTION_DRIVER, hiveConf)); argList.add("-n"); argList.add(userName); argList.add("-p"); @@ -384,14 +355,6 @@ public void runBeeLine(String sqlScriptFile) throws IOException { } } - private String getValidConfVar(ConfVars confVar) throws IOException { - String confVarStr = hiveConf.get(confVar.varname); - if (confVarStr == null || confVarStr.isEmpty()) { - throw new IOException("Empty " + confVar.varname); - } - return confVarStr; - } - // Create the required command line options @SuppressWarnings("static-access") private static void initOptions(Options cmdLineOptions) { @@ -423,6 +386,9 @@ private static void initOptions(Options cmdLineOptions) { Option dbTypeOpt = OptionBuilder.withArgName("databaseType") .hasArgs().withDescription("Metastore database type") .create("dbType"); + Option dbOpts = OptionBuilder.withArgName("databaseOpts") + .hasArgs().withDescription("Backend DB specific options") + .create("dbOpts"); Option dryRunOpt = new Option("dryRun", "list SQL scripts (no execute)"); Option verboseOpt = new Option("verbose", "only print SQL statements"); @@ -432,6 +398,7 @@ private static void initOptions(Options cmdLineOptions) { cmdLineOptions.addOption(passwdOpt); cmdLineOptions.addOption(dbTypeOpt); cmdLineOptions.addOption(verboseOpt); + cmdLineOptions.addOption(dbOpts); cmdLineOptions.addOptionGroup(optGroup); } @@ -488,7 +455,9 @@ public static void main(String[] args) { if (line.hasOption("verbose")) { schemaTool.setVerbose(true); } - + if (line.hasOption("dbOpts")) { + schemaTool.setDbOpts(line.getOptionValue("dbOpts")); + } if (line.hasOption("info")) { schemaTool.showInfo(); } else if (line.hasOption("upgradeSchema")) { diff --git a/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java b/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java index 6bbb046..a2c18c7 100644 --- a/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java +++ b/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java @@ -51,11 +51,14 @@ private void updateCsvPreference() { char newDel = beeLine.getOpts().getDelimiterForDSV(); // if delimiter changed, rebuild the csv preference if (newDel != curDel) { + // "" is passed as the end of line symbol in following function, as + // beeline itself adds newline csvPreference = new CsvPreference.Builder('"', newDel, "").build(); } } } + @Override public int print(Rows rows) { updateCsvPreference(); diff --git a/beeline/src/main/resources/BeeLine.properties b/beeline/src/main/resources/BeeLine.properties index d40c0dc..d038d46 100644 --- a/beeline/src/main/resources/BeeLine.properties +++ b/beeline/src/main/resources/BeeLine.properties @@ -68,7 +68,7 @@ help-procedures: List all the procedures help-tables: List all the tables in the database help-columns: List all the columns for the specified table help-properties: Connect to the database specified in the properties file(s) -help-outputformat: Set the output format for displaying results (table,vertical,csv,dsv,tsv,xmlattrs,xmlelements) +help-outputformat: Set the output format for displaying results (table,vertical,csv2,dsv,tsv2,xmlattrs,xmlelements, and deprecated formats(csv, tsv)) help-delimiterForDSV: Set the delimiter for dsv output format help-nullemptystring: Set to true to get historic behavior of printing null as empty string. Default is false. @@ -167,7 +167,8 @@ cmd-usage: Usage: java org.apache.hive.cli.beeline.BeeLine \n \ \ --maxColumnWidth=MAXCOLWIDTH the maximum width to use when displaying columns\n \ \ --silent=[true/false] be more silent\n \ \ --autosave=[true/false] automatically save preferences\n \ -\ --outputformat=[table/vertical/csv/tsv/dsv] format mode for result display\n \ +\ --outputformat=[table/vertical/csv2/tsv2/dsv/csv/tsv] format mode for result display\n \ +\ Note that csv, and tsv are deprecated - use csv2, tsv2 instead\n\ \ --truncateTable=[true/false] truncate table column when it exceeds length\n \ \ --delimiterForDSV=DELIMITER specify the delimiter for delimiter-separated values output format (default: |)\n \ \ --isolation=LEVEL set the transaction isolation level\n \ diff --git a/bin/ext/hiveserver2.cmd b/bin/ext/hiveserver2.cmd index a5f3bb5..10895a7 100644 --- a/bin/ext/hiveserver2.cmd +++ b/bin/ext/hiveserver2.cmd @@ -56,9 +56,19 @@ if defined HBASE_HOME ( ) if [%1]==[hiveserver2_help] goto :hiveserver2_help +<<<<<<< if [%1]==[hiveserver2_catservice] goto :hiveserver2_catservice +======= + +if [%1]==[hiveserver2_catservice] goto :hiveserver2_catservice + +:hiveserver2 + + @rem hadoop 20 or newer - skip the aux_jars option and hiveconf + call %HIVE_BIN_PATH%\ext\util\execHiveCmd.cmd %CLASS% +>>>>>>> if [%1]==[hiveserver2_catcmd] goto :hiveserver2_catcmd :hiveserver2 @@ -75,6 +85,20 @@ goto :EOF :hiveserver2_catservice @echo ^ +<<<<<<< +======= +@echo ^HiveServer2^ +@echo ^HiveServer2^ +@echo ^Hadoop HiveServer2 Service^ +@echo ^%JAVA_HOME%\bin\java^ +@echo ^%JAVA_HEAP_MAX% %HADOOP_OPTS% -classpath %CLASSPATH%;%HIVE_HBASE_PATH% %CLASS% -hiveconf hive.hadoop.classpath=%HIVE_LIB%\* -hiveconf hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory -hiveconf hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator -hiveconf hive.metastore.uris=" " %HIVE_OPTS%^ +@echo ^ +goto :EOF + +:AddToHiveHbasePath +if not defined HIVE_HBASE_PATH ( + set HIVE_HBASE_PATH=%1 +>>>>>>> @echo ^HiveServer2^ @echo ^HiveServer2^ @echo ^Hadoop HiveServer2 Service^ diff --git a/bin/ext/hiveserver2.cmd.orig b/bin/ext/hiveserver2.cmd.orig new file mode 100644 index 0000000..a5f3bb5 --- /dev/null +++ b/bin/ext/hiveserver2.cmd.orig @@ -0,0 +1,103 @@ +@echo off +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. + +set CLASS=org.apache.hive.service.server.HiveServer2 +pushd %HIVE_LIB% +for /f %%a IN ('dir /b hive-service-*.jar') do ( + set JAR=%HIVE_LIB%\%%a +) +popd + +if defined HBASE_HOME ( + dir %HBASE_HOME%\lib > nul + if %errorlevel%==0 ( + pushd %HBASE_HOME%\lib + set HIVE_HBASE_PATH= + for /f %%a IN ('dir /b hbase-server-**-hadoop2.jar') do ( + call :AddToHiveHbasePath %HBASE_HOME%\lib\%%a + ) + for /f %%a IN ('dir /b hbase-client-**-hadoop2.jar') do ( + call :AddToHiveHbasePath %HBASE_HOME%\lib\%%a + ) + for /f %%a IN ('dir /b hbase-protocol-**-hadoop2.jar') do ( + call :AddToHiveHbasePath %HBASE_HOME%\lib\%%a + ) + for /f %%a IN ('dir /b htrace-core-**.jar') do ( + call :AddToHiveHbasePath %HBASE_HOME%\lib\%%a + ) + for /f %%a IN ('dir /b hbase-common-**-hadoop2.jar') do ( + call :AddToHiveHbasePath %HBASE_HOME%\lib\%%a + ) + for /f %%a IN ('dir /b hbase-hadoop-compat-**-hadoop2.jar') do ( + call :AddToHiveHbasePath %HBASE_HOME%\lib\%%a + ) + ) + popd + if defined HBASE_CONF_DIR ( + dir %HBASE_CONF_DIR% > nul + if %errorlevel%==0 ( + call :AddToHiveHbasePath %HBASE_CONF_DIR% + ) + ) + ) +) + +if [%1]==[hiveserver2_help] goto :hiveserver2_help + +if [%1]==[hiveserver2_catservice] goto :hiveserver2_catservice + +if [%1]==[hiveserver2_catcmd] goto :hiveserver2_catcmd + +:hiveserver2 + echo "Starting Hive Thrift Server" + + @rem hadoop 20 or newer - skip the aux_jars option and hiveconf + call %HIVE_BIN_PATH%\ext\util\execHiveCmd.cmd %CLASS% +goto :EOF + +:hiveserver2_help + set HIVEARGS=-h + goto :hiveserver2 +goto :EOF + +:hiveserver2_catservice +@echo ^ +@echo ^HiveServer2^ +@echo ^HiveServer2^ +@echo ^Hadoop HiveServer2 Service^ +@echo ^%SystemRoot%\system32\cmd.exe^ +@echo ^/c %HIVE_BIN_PATH%\ext\hs2service.cmd ^ +@echo ^ +goto :EOF + + +:hiveserver2_catcmd +if not defined HADOOP_CLASSPATH ( + @echo set HADOOP_CLASSPATH=%HIVE_LIB%\* + ) else ( + @echo set HADOOP_CLASSPATH=%HADOOP_CLASSPATH%;%HIVE_LIB%\* + ) +@echo %JAVA_HOME%\bin\java %JAVA_HEAP_MAX% %HADOOP_OPTS% -classpath %CLASSPATH%;%HIVE_HBASE_PATH% %CLASS% -hiveconf hive.metastore.uris=" " -hiveconf hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory -hiveconf hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator %HIVE_OPTS% +goto :EOF + +:AddToHiveHbasePath +if not defined HIVE_HBASE_PATH ( + set HIVE_HBASE_PATH=%1 + ) else ( + set HIVE_HBASE_PATH=%HIVE_HBASE_PATH%;%1 + ) +) +goto :EOF diff --git a/bin/ext/hiveserver2.sh b/bin/ext/hiveserver2.sh index 45d7893..42d3d79 100644 --- a/bin/ext/hiveserver2.sh +++ b/bin/ext/hiveserver2.sh @@ -17,7 +17,6 @@ THISSERVICE=hiveserver2 export SERVICE_LIST="${SERVICE_LIST}${THISSERVICE} " hiveserver2() { - echo "Starting HiveServer2" CLASS=org.apache.hive.service.server.HiveServer2 if $cygwin; then HIVE_LIB=`cygpath -w "$HIVE_LIB"` diff --git a/bin/hive b/bin/hive index 4063632..da74351 100755 --- a/bin/hive +++ b/bin/hive @@ -240,7 +240,9 @@ if [[ -n $HBASE_BIN ]] ; then fi if [ "${AUX_PARAM}" != "" ]; then - HIVE_OPTS="$HIVE_OPTS --hiveconf hive.aux.jars.path=${AUX_PARAM}" + if [[ "$SERVICE" != beeline ]]; then + HIVE_OPTS="$HIVE_OPTS --hiveconf hive.aux.jars.path=${AUX_PARAM}" + fi AUX_JARS_CMD_LINE="-libjars ${AUX_PARAM}" fi diff --git a/bin/hive.cmd b/bin/hive.cmd index c2e9853..ea2ff14 100644 --- a/bin/hive.cmd +++ b/bin/hive.cmd @@ -281,9 +281,18 @@ if defined HELP ( if defined CATSERVICE ( if exist %HADOOP_HOME%\libexec\hadoop-config.cmd ( call %HADOOP_HOME%\libexec\hadoop-config.cmd +<<<<<<< ) else ( call %HADOOP_HOME%\libexec\hadoop-config.cmd ) +======= + ) else ( + call %HADOOP_HOME%\libexec\hadoop-config.cmd + ) + call %HIVE_BIN_PATH%\ext\%TORUN%.cmd %TORUN%%CATSERVICE% %* + goto :EOF +) +>>>>>>> if %TORUN% == hiveserver2 ( call %HIVE_BIN_PATH%\ext\hiveserver2.cmd hiveserver2_catcmd > %HIVE_BIN_PATH%\ext\hs2service.cmd ) diff --git a/bin/hive.cmd.orig b/bin/hive.cmd.orig new file mode 100644 index 0000000..c2e9853 --- /dev/null +++ b/bin/hive.cmd.orig @@ -0,0 +1,385 @@ +@echo off +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +SetLocal EnableDelayedExpansion + +@rem Set the path + +if not defined HIVE_BIN_PATH ( + set HIVE_BIN_PATH=%~dp0 +) + +if "%HIVE_BIN_PATH:~-1%" == "\" ( + set HIVE_BIN_PATH=%HIVE_BIN_PATH:~0,-1% +) + +set HIVE_CONFIG_SCRIPT=%HIVE_BIN_PATH%\hive-config.cmd + +if exist %HIVE_CONFIG_SCRIPT% ( + CALL %HIVE_CONFIG_SCRIPT% %* +) + +set SERVICE= +set HELP= +set CATSERVICE= +set DEBUG= +set CURRENTARG= +set HIVEARGS= +rem set AUX_CLASSPATH= +set AUX_PARAM= + +@rem parse the command line arguments +:ProcessCmdLine + if [%1]==[] goto :FinishArgs + + set temp=%1 + set temp=%temp:~0, 7% + + if %temp%==--debug ( + set DEBUG=%* + shift + goto :ProcessCmdLine + ) + + if %1==--config ( + shift + shift + goto :ProcessCmdLine + ) + + if %1==--auxpath ( + shift + shift + goto :ProcessCmdLine + ) + + if %1==--service ( + set SERVICE=%2 + + if [%3]==[catservicexml] ( + set CATSERVICE=_catservice + shift + ) + shift + shift + goto :ProcessCmdLine + ) + + if %1==--rcfilecat ( + set SERVICE=rcfilecat + shift + goto :ProcessCmdLine + ) + + if %1==--orcfiledump ( + set SERVICE=orcfiledump + shift + goto :ProcessCmdLine + ) + + if %1==--help ( + set HELP=_help + shift + goto :ProcessCmdLine + ) + + @rem parameter at %1 does not match any option, these are optional params + goto :FinishArgs +:FinishArgs + +if defined DEBUG ( + if defined HELP ( + call %HIVE_BIN_PATH%\ext\debug.cmd HELP + goto :EOF + ) + + call %HIVE_BIN_PATH%\ext\debug.cmd %DEBUG% +) + +if defined HIVE_MAIN_CLIENT_DEBUG_OPTS ( + set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_CLIENT_OPTS% %HIVE_MAIN_CLIENT_DEBUG_OPTS% +) + +if not [%1]==[] ( + set CURRENTARG=%1 + call :MakeHiveArgs %* +) + +if not defined SERVICE ( + if defined HELP ( + set SERVICE=help + ) else ( + set SERVICE=cli + ) +) + +if not defined HIVE_HOME ( + echo "HIVE_HOME needs to be defined to point at the root of the hive install" + exit /b 1 +) + +if not defined HIVE_CONF_DIR ( + set HIVE_CONF_DIR=%HIVE_HOME%\conf +) + +if exist %HIVE_CONF_DIR%/hive-env.cmd CALL %HIVE_CONF_DIR%/hive-env.cmd + +@rem sort out classpath and make sure dependencies exist +set CLASSPATH=%HIVE_CONF_DIR% + +set HIVE_LIB=%HIVE_HOME%\lib + +@rem needed for execution +if not exist %HIVE_LIB%\hive-exec-*.jar ( + echo "Missing Hive Execution Jar: %HIVE_LIB%/hive-exec-*.jar" + exit /b 1 +) + +if not exist %HIVE_LIB%\hive-metastore-*.jar ( + echo "Missing Hive MetaStore Jar" + exit /b 1 +) + +@rem cli specific code +if not exist %HIVE_LIB%\hive-cli-*.jar ( + echo "Missing Hive CLI Jar" + exit /b 1 +) + +set CLASSPATH=%CLASSPATH%;%HIVE_LIB%\* + +@rem maybe we should just make users set HADOOP_HOME env variable as a prereq +@rem in the next iteration, use "where" command to find directory of hadoop install from path +if not defined HADOOP_HOME ( + echo "HADOOP_HOME needs to be defined to point at the hadoop installation" + exit /b 1 +) + +@rem supress the HADOOP_HOME warnings in 1.x.x +set HADOOP_HOME_WARN_SUPPRESS=true + +set HADOOP=%HADOOP_HOME%\bin\hadoop.cmd +if not exist %HADOOP% ( + echo "Missing hadoop installation: %HADOOP_HOME% must be set" + exit /b 1 +) + +@rem can only run against hadoop 1.0.0 as prereq for this iteration - can't figure out the regex/awk script to determine compatibility + +@rem add auxilary jars such as serdes +if not defined HIVE_AUX_JARS_PATH goto :AddAuxLibDir + +setLocal EnableDelayedExpansion +:auxJarLoop + for /f "delims=," %%a in ("!HIVE_AUX_JARS_PATH!") do ( + set auxjar=%%a + if exist %%a ( + if exist "%%a\nul" ( + @rem %%a is a dir + pushd %%a + for /f %%b IN ('dir /b *.jar') do ( + set AUX_CLASSPATH=!AUX_CLASSPATH!;%%a\%%b + call :AddToAuxParam %%a\%%b + ) + popd + ) else ( + @rem %%a is a file + set AUX_CLASSPATH=!AUX_CLASSPATH!;%%a + call :AddToAuxParam %%a + ) + ) + ) + :striploop + set stripchar=!HIVE_AUX_JARS_PATH:~0,1! + set HIVE_AUX_JARS_PATH=!HIVE_AUX_JARS_PATH:~1! + if "!HIVE_AUX_JARS_PATH!" EQU "" goto auxJarLoopEnd + if "!stripchar!" NEQ "," goto striploop + goto auxJarLoop + +:auxJarLoopEnd + +if defined HIVE_AUX_JARS_PATH ( + echo "setting aux param %HIVE_AUX_JARS_PATH%" + set AUX_CLASSPATH=%HIVE_AUX_JARS_PATH% + set AUX_PARAM=file://%HIVE_AUX_JARS_PATH% +) + + +:AddAuxLibDir +@rem adding jars from auxlib directory +if exist %HIVE_HOME%\auxlib ( + pushd %HIVE_HOME%\auxlib + for /f %%a IN ('dir /b *.jar') do ( + set AUX_CLASSPATH=%AUX_CLASSPATH%;%%a + call :AddToAuxParam %%a + ) + popd +) + +@rem pass classpath to hadoop +set HADOOP_CLASSPATH=%HADOOP_CLASSPATH%;%CLASSPATH%;%AUX_CLASSPATH% + +@rem also pass hive classpath to hadoop +if defined HIVE_CLASSPATH ( + set HADOOP_CLASSPATH=%HADOOP_CLASSPATH%;%HIVE_CLASSPATH% +) + +@rem set hbase components +if defined HBASE_HOME ( + if not defined HBASE_CONF_DIR ( + if exist %HBASE_HOME%\conf ( + set HBASE_CONF_DIR=%HBASE_HOME%\conf + ) + ) + if defined HBASE_CONF_DIR ( + call :AddToHadoopClassPath %HBASE_CONF_DIR% + ) + if exist %HBASE_HOME%\lib ( + call :AddToHadoopClassPath %HBASE_HOME%\lib\* + ) +) + +if defined AUX_PARAM ( + set HIVE_OPTS=%HIVE_OPTS% -hiveconf hive.aux.jars.path="%AUX_PARAM%" + set AUX_JARS_CMD_LINE="-libjars %AUX_PARAM%" +) + +@rem Get ready to run the services +set SERVICE_COUNT=0 +set TORUN="" +call :AddServices +For /L %%i in (1,1,%SERVICE_COUNT%) do ( + if "%SERVICE%" == "!VAR%%i!" ( + set TORUN=!VAR%%i! + ) +) + +if %TORUN% == "" ( + echo "Service %SERVICE% not available" + exit /b 1 +) + + +if defined HELP ( + call %HIVE_BIN_PATH%\ext\%TORUN%.cmd %TORUN%%HELP% %* + goto :EOF +) + +@rem generate xml for the service, also append hadoop dependencies to the classpath +if defined CATSERVICE ( + if exist %HADOOP_HOME%\libexec\hadoop-config.cmd ( + call %HADOOP_HOME%\libexec\hadoop-config.cmd + ) else ( + call %HADOOP_HOME%\libexec\hadoop-config.cmd + ) + if %TORUN% == hiveserver2 ( + call %HIVE_BIN_PATH%\ext\hiveserver2.cmd hiveserver2_catcmd > %HIVE_BIN_PATH%\ext\hs2service.cmd + ) + call %HIVE_BIN_PATH%\ext\%TORUN%.cmd %TORUN%%CATSERVICE% %* + goto :EOF +) + +call %HIVE_BIN_PATH%\ext\%TORUN%.cmd %TORUN% %* + + + +goto :EOF +@rem done body of script + + +@rem start utility functions here + +@rem strip off preceding arguments like --service so that subsequent args can be passed on +:MakeHiveArgs + set _count=0 + set _shift=1 + set HIVEARGS= + + if not defined CURRENTARG ( + goto :EndLoop + ) + :HiveArgsLoop + if [%1]==[] ( + goto :EndLoop + ) + if not %1==%CURRENTARG% ( + shift + goto :HiveArgsLoop + ) + + if not defined HIVEARGS ( + set HIVEARGS=%1 + ) else ( + set HIVEARGS=%HIVEARGS% %1 + ) + shift + set CURRENTARG=%1 + goto :HiveArgsLoop + :EndLoop +goto :EOF + +@rem makes list of available services +:AddServices + set /a SERVICE_COUNT = %SERVICE_COUNT% + 1 + set VAR%SERVICE_COUNT%=cli + + set /a SERVICE_COUNT = %SERVICE_COUNT% + 1 + set VAR%SERVICE_COUNT%=help + + set /a SERVICE_COUNT = %SERVICE_COUNT% + 1 + set VAR%SERVICE_COUNT%=hiveserver + + set /a SERVICE_COUNT = %SERVICE_COUNT% + 1 + set VAR%SERVICE_COUNT%=hiveserver2 + + set /a SERVICE_COUNT = %SERVICE_COUNT% + 1 + set VAR%SERVICE_COUNT%=hwi + + set /a SERVICE_COUNT = %SERVICE_COUNT% + 1 + set VAR%SERVICE_COUNT%=jar + + set /a SERVICE_COUNT = %SERVICE_COUNT% + 1 + set VAR%SERVICE_COUNT%=lineage + + set /a SERVICE_COUNT = %SERVICE_COUNT% + 1 + set VAR%SERVICE_COUNT%=metastore + + set /a SERVICE_COUNT = %SERVICE_COUNT% + 1 + set VAR%SERVICE_COUNT%=rcfilecat + + set /a SERVICE_COUNT = %SERVICE_COUNT% + 1 + set VAR%SERVICE_COUNT%=orcfiledump + + set /a SERVICE_COUNT = %SERVICE_COUNT% + 1 + set VAR%SERVICE_COUNT%=schematool +goto :EOF + +:AddToAuxParam +if not defined AUX_PARAM ( + set AUX_PARAM=file:///%1 + ) else ( + set AUX_PARAM=%AUX_PARAM%,file:///%1 + ) +) +goto :EOF + +:AddToHadoopClassPath +if defined HADOOP_CLASSPATH ( + set HADOOP_CLASSPATH=%HADOOP_CLASSPATH%;%1 +) else ( + set HADOOP_CLASSPATH=%1 + ) +) +goto :EOF diff --git a/cli/pom.xml b/cli/pom.xml index 345f124..4b92bc2 100644 --- a/cli/pom.xml +++ b/cli/pom.xml @@ -19,7 +19,7 @@ org.apache.hive hive - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../pom.xml diff --git a/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java b/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java index 17ef2db..d7a9b0e 100644 --- a/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java +++ b/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java @@ -60,6 +60,7 @@ import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.exec.mr.HadoopJobExecHelper; +import org.apache.hadoop.hive.ql.exec.tez.TezJobExecHelper; import org.apache.hadoop.hive.ql.parse.HiveParser; import org.apache.hadoop.hive.ql.parse.VariableSubstitution; import org.apache.hadoop.hive.ql.processors.CommandProcessor; @@ -125,7 +126,7 @@ public int processCmd(String cmd) { ret = 1; } else { try { - this.processFile(cmd_1); + ret = processFile(cmd_1); } catch (IOException e) { console.printError("Failed processing file "+ cmd_1 +" "+ e.getLocalizedMessage(), stringifyException(e)); @@ -384,6 +385,7 @@ public void handle(Signal signal) { // First, kill any running MR jobs HadoopJobExecHelper.killRunningJobs(); + TezJobExecHelper.killRunningJobs(); HiveInterruptUtils.interrupt(); } }); diff --git a/cli/src/test/org/apache/hadoop/hive/cli/TestRCFileCat.java b/cli/src/test/org/apache/hadoop/hive/cli/TestRCFileCat.java index 7ee02e8..11ceb31 100644 --- a/cli/src/test/org/apache/hadoop/hive/cli/TestRCFileCat.java +++ b/cli/src/test/org/apache/hadoop/hive/cli/TestRCFileCat.java @@ -25,6 +25,7 @@ import java.io.File; import java.io.IOException; import java.io.PrintStream; +import java.net.URI; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -89,19 +90,19 @@ public void testRCFileCat() throws Exception { try { - String[] params = {"--verbose","file://" + template.getAbsolutePath() }; + String[] params = {"--verbose","file://" + template.toURI().getPath() }; assertEquals(0, fileCat.run(params)); assertTrue(dataOut.toString().contains("123\t456\t789\t1000\t5.3\thive and hadoop\t\tNULL")); assertTrue(dataOut.toString().contains("100\t200\t123\t1000\t5.3\thive and hadoop\t\tNULL")); assertTrue(dataOut.toString().contains("200\t400\t678\t1000\t4.8\thive and hadoop\t\tTEST")); dataOut.reset(); - params = new String[] { "--start=-10","--file-sizes","file://" + template.getAbsolutePath() }; + params = new String[] { "--start=-10","--file-sizes", "file://" + template.toURI().getPath() }; assertEquals(0, fileCat.run(params)); assertTrue(dataOut.toString().contains("File size (uncompressed): 105. File size (compressed): 134. Number of rows: 3.")); dataOut.reset(); - params = new String[] {"--start=0", "--column-sizes","file://" + template.getAbsolutePath() }; + params = new String[] {"--start=0", "--column-sizes","file://" + template.toURI().getPath() }; assertEquals(0, fileCat.run(params)); assertTrue(dataOut.toString().contains("0\t9\t17")); assertTrue(dataOut.toString().contains("1\t9\t17")); @@ -112,7 +113,8 @@ public void testRCFileCat() throws Exception { dataOut.reset(); - params = new String[] {"--start=0", "--column-sizes-pretty","file://" + template.getAbsolutePath() }; + params = new String[] {"--start=0", "--column-sizes-pretty", + "file://" + template.toURI().getPath() }; assertEquals(0, fileCat.run(params)); assertTrue(dataOut.toString().contains("Column 0: Uncompressed size: 9 Compressed size: 17")); assertTrue(dataOut.toString().contains("Column 1: Uncompressed size: 9 Compressed size: 17")); @@ -127,7 +129,8 @@ public void testRCFileCat() throws Exception { "[--column-sizes | --column-sizes-pretty] [--file-sizes] fileName")); dataErr.reset(); - params = new String[] { "--fakeParameter","file://" + template.getAbsolutePath()}; + params = new String[] { "--fakeParameter", + "file://" + template.toURI().getPath()}; assertEquals(-1, fileCat.run(params)); assertTrue(dataErr.toString().contains("RCFileCat [--start=start_offet] [--length=len] [--verbose] " + "[--column-sizes | --column-sizes-pretty] [--file-sizes] fileName")); diff --git a/common/pom.xml b/common/pom.xml index 01c74ba..76d0fe7 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -19,7 +19,7 @@ org.apache.hive hive - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../pom.xml diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 5e180a6..6120b1d 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -53,6 +53,8 @@ import org.apache.hadoop.util.Shell; import org.apache.hive.common.HiveCompat; +import com.google.common.base.Joiner; + /** * Hive Configuration. */ @@ -75,11 +77,9 @@ private static final Map metaConfs = new HashMap(); private final List restrictList = new ArrayList(); - private boolean isWhiteListRestrictionEnabled = false; - private final List modWhiteList = new ArrayList(); + private Pattern modWhiteListPattern = null; private boolean isSparkConfigUpdated = false; - public boolean getSparkConfigUpdated() { return isSparkConfigUpdated; } @@ -249,9 +249,9 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { "The compression codec and other options are determined from Hadoop config variables mapred.output.compress*"), COMPRESSINTERMEDIATECODEC("hive.intermediate.compression.codec", "", ""), COMPRESSINTERMEDIATETYPE("hive.intermediate.compression.type", "", ""), - BYTESPERREDUCER("hive.exec.reducers.bytes.per.reducer", (long) (1000 * 1000 * 1000), - "size per reducer.The default is 1G, i.e if the input size is 10G, it will use 10 reducers."), - MAXREDUCERS("hive.exec.reducers.max", 999, + BYTESPERREDUCER("hive.exec.reducers.bytes.per.reducer", (long) (256 * 1000 * 1000), + "size per reducer.The default is 256Mb, i.e if the input size is 1G, it will use 4 reducers."), + MAXREDUCERS("hive.exec.reducers.max", 1009, "max number of reducers will be used. If the one specified in the configuration parameter mapred.reduce.tasks is\n" + "negative, Hive will use this one as the max number of reducers when automatically determine number of reducers."), PREEXECHOOKS("hive.exec.pre.hooks", "", @@ -281,8 +281,9 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { DYNAMICPARTITIONING("hive.exec.dynamic.partition", true, "Whether or not to allow dynamic partitions in DML/DDL."), DYNAMICPARTITIONINGMODE("hive.exec.dynamic.partition.mode", "strict", - "In strict mode, the user must specify at least one static partition \n" + - "in case the user accidentally overwrites all partitions."), + "In strict mode, the user must specify at least one static partition\n" + + "in case the user accidentally overwrites all partitions.\n" + + "In nonstrict mode all partitions are allowed to be dynamic."), DYNAMICPARTITIONMAXPARTS("hive.exec.max.dynamic.partitions", 1000, "Maximum number of dynamic partitions allowed to be created in total."), DYNAMICPARTITIONMAXPARTSPERNODE("hive.exec.max.dynamic.partitions.pernode", 100, @@ -321,7 +322,7 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { "When hive.exec.mode.local.auto is true, the number of tasks should less than this for local mode."), DROPIGNORESNONEXISTENT("hive.exec.drop.ignorenonexistent", true, - "Do not report an error if DROP TABLE/VIEW specifies a non-existent table/view"), + "Do not report an error if DROP TABLE/VIEW/Index specifies a non-existent table/view/index"), HIVEIGNOREMAPJOINHINT("hive.ignore.mapjoin.hint", true, "Ignore the mapjoin hint"), @@ -422,13 +423,19 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { "The delegation token store implementation. Set to org.apache.hadoop.hive.thrift.ZooKeeperTokenStore for load-balanced cluster."), METASTORE_CLUSTER_DELEGATION_TOKEN_STORE_ZK_CONNECTSTR( "hive.cluster.delegation.token.store.zookeeper.connectString", "", - "The ZooKeeper token store connect string."), + "The ZooKeeper token store connect string. You can re-use the configuration value\n" + + "set in hive.zookeeper.quorum, by leaving this parameter unset."), METASTORE_CLUSTER_DELEGATION_TOKEN_STORE_ZK_ZNODE( - "hive.cluster.delegation.token.store.zookeeper.znode", "/hive/cluster/delegation", - "The root path for token store data."), + "hive.cluster.delegation.token.store.zookeeper.znode", "/hivedelegation", + "The root path for token store data. Note that this is used by both HiveServer2 and\n" + + "MetaStore to store delegation Token. One directory gets created for each of them.\n" + + "The final directory names would have the servername appended to it (HIVESERVER2,\n" + + "METASTORE)."), METASTORE_CLUSTER_DELEGATION_TOKEN_STORE_ZK_ACL( "hive.cluster.delegation.token.store.zookeeper.acl", "", - "ACL for token store entries. List comma separated all server principals for the cluster."), + "ACL for token store entries. Comma separated list of ACL entries. For example:\n" + + "sasl:hive/host1@MY.DOMAIN:cdrwa,sasl:hive/host2@MY.DOMAIN:cdrwa\n" + + "Defaults to all permissions for the hiveserver2/metastore process user."), METASTORE_CACHE_PINOBJTYPES("hive.metastore.cache.pinobjtypes", "Table,StorageDescriptor,SerDeInfo,Partition,Database,Type,FieldSchema,Order", "List of comma separated metastore object types that should be pinned in the cache"), METASTORE_CONNECTION_POOLING_TYPE("datanucleus.connectionPoolingType", "BONECP", @@ -501,20 +508,33 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { "However, it doesn't work correctly with integral values that are not normalized (e.g. have\n" + "leading zeroes, like 0012). If metastore direct SQL is enabled and works, this optimization\n" + "is also irrelevant."), - METASTORE_TRY_DIRECT_SQL("hive.metastore.try.direct.sql", true, ""), - METASTORE_TRY_DIRECT_SQL_DDL("hive.metastore.try.direct.sql.ddl", true, ""), + METASTORE_TRY_DIRECT_SQL("hive.metastore.try.direct.sql", true, + "Whether the Hive metastore should try to use direct SQL queries instead of the\n" + + "DataNucleus for certain read paths. This can improve metastore performance when\n" + + "fetching many partitions or column statistics by orders of magnitude; however, it\n" + + "is not guaranteed to work on all RDBMS-es and all versions. In case of SQL failures,\n" + + "the metastore will fall back to the DataNucleus, so it's safe even if SQL doesn't\n" + + "work for all queries on your datastore. If all SQL queries fail (for example, your\n" + + "metastore is backed by MongoDB), you might want to disable this to save the\n" + + "try-and-fall-back cost."), + METASTORE_TRY_DIRECT_SQL_DDL("hive.metastore.try.direct.sql.ddl", true, + "Same as hive.metastore.try.direct.sql, for read statements within a transaction that\n" + + "modifies metastore data. Due to non-standard behavior in Postgres, if a direct SQL\n" + + "select query has incorrect syntax or something similar inside a transaction, the\n" + + "entire transaction will fail and fall-back to DataNucleus will not be possible. You\n" + + "should disable the usage of direct SQL inside transactions if that happens in your case."), METASTORE_DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES( "hive.metastore.disallow.incompatible.col.type.changes", false, - "If true (default is false), ALTER TABLE operations which change the type of \n" + - "a column (say STRING) to an incompatible type (say MAP) are disallowed. \n" + + "If true (default is false), ALTER TABLE operations which change the type of a\n" + + "column (say STRING) to an incompatible type (say MAP) are disallowed.\n" + "RCFile default SerDe (ColumnarSerDe) serializes the values in such a way that the\n" + "datatypes can be converted from string to any type. The map is also serialized as\n" + - "a string, which can be read as a string as well. However, with any binary \n" + + "a string, which can be read as a string as well. However, with any binary\n" + "serialization, this is not true. Blocking the ALTER TABLE prevents ClassCastExceptions\n" + - "when subsequently trying to access old partitions. \n" + + "when subsequently trying to access old partitions.\n" + "\n" + - "Primitive types like INT, STRING, BIGINT, etc are compatible with each other and are \n" + - "not blocked. \n" + + "Primitive types like INT, STRING, BIGINT, etc., are compatible with each other and are\n" + + "not blocked.\n" + "\n" + "See HIVE-4409 for more details."), @@ -587,8 +607,10 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { HIVEJOBNAMELENGTH("hive.jobname.length", 50, "max jobname length"), // hive jar - HIVEJAR("hive.jar.path", "", ""), - HIVEAUXJARS("hive.aux.jars.path", "", ""), + HIVEJAR("hive.jar.path", "", + "The location of hive_cli.jar that is used when submitting jobs in a separate jvm."), + HIVEAUXJARS("hive.aux.jars.path", "", + "The location of the plugin jars that contain implementations of user defined functions and serdes."), // reloadable jars HIVERELOADABLEJARS("hive.reloadable.aux.jars.path", "", @@ -596,9 +618,9 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { + "used as the auxiliary classes like creating a UDF or SerDe."), // hive added files and jars - HIVEADDEDFILES("hive.added.files.path", "", ""), - HIVEADDEDJARS("hive.added.jars.path", "", ""), - HIVEADDEDARCHIVES("hive.added.archives.path", "", ""), + HIVEADDEDFILES("hive.added.files.path", "", "This an internal parameter."), + HIVEADDEDJARS("hive.added.jars.path", "", "This an internal parameter."), + HIVEADDEDARCHIVES("hive.added.archives.path", "", "This an internal parameter."), HIVE_CURRENT_DATABASE("hive.current.database", "", "Database name used by current session. Internal usage only.", true), @@ -607,8 +629,6 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { new TimeValidator(TimeUnit.SECONDS), "How long to run autoprogressor for the script/UDTF operators.\n" + "Set to 0 for forever."), - HIVETABLENAME("hive.table.name", "", ""), - HIVEPARTITIONNAME("hive.partition.name", "", ""), HIVESCRIPTAUTOPROGRESS("hive.script.auto.progress", false, "Whether Hive Transform/Map/Reduce Clause should automatically send progress information to TaskTracker \n" + "to avoid the task getting killed because of inactivity. Hive sends progress information when the script is \n" + @@ -619,6 +639,10 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { "transform function (the custom mapper/reducer that the user has specified in the query)"), HIVESCRIPTTRUNCATEENV("hive.script.operator.truncate.env", false, "Truncate each environment variable for external script in scripts operator to 20KB (to fit system limits)"), + HIVESCRIPT_ENV_BLACKLIST("hive.script.operator.env.blacklist", + "hive.txn.valid.txns,hive.script.operator.env.blacklist", + "Comma separated list of keys from the configuration file not to convert to environment " + + "variables when envoking the script operator"), HIVEMAPREDMODE("hive.mapred.mode", "nonstrict", "The mode in which the Hive operations are being performed. \n" + "In strict mode, some risky queries are not allowed to run. They include:\n" + @@ -650,7 +674,7 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { "How many rows in the joining tables (except the streaming table) should be cached in memory."), // CBO related - HIVE_CBO_ENABLED("hive.cbo.enable", false, "Flag to control enabling Cost Based Optimizations using Optiq framework."), + HIVE_CBO_ENABLED("hive.cbo.enable", false, "Flag to control enabling Cost Based Optimizations using Calcite framework."), // hive.mapjoin.bucket.cache.size has been replaced by hive.smbjoin.cache.row, // need to remove by hive .13. Also, do not change default (see SMB operator) @@ -820,10 +844,10 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { "if hive.merge.mapfiles is true, and for map-reduce jobs if hive.merge.mapredfiles is true."), HIVEMERGERCFILEBLOCKLEVEL("hive.merge.rcfile.block.level", true, ""), HIVEMERGEORCFILESTRIPELEVEL("hive.merge.orcfile.stripe.level", true, - "When hive.merge.mapfiles or hive.merge.mapredfiles is enabled while writing a\n" + - " table with ORC file format, enabling this config will do stripe level fast merge\n" + - " for small ORC files. Note that enabling this config will not honor padding tolerance\n" + - " config (hive.exec.orc.block.padding.tolerance)."), + "When hive.merge.mapfiles, hive.merge.mapredfiles or hive.merge.tezfiles is enabled\n" + + "while writing a table with ORC file format, enabling this config will do stripe-level\n" + + "fast merge for small ORC files. Note that enabling this config will not honor the\n" + + "padding tolerance config (hive.exec.orc.block.padding.tolerance)."), HIVEUSEEXPLICITRCFILEHEADER("hive.exec.rcfile.use.explicit.header", true, "If this is set the header for RCFiles will simply be RCF. If this is not\n" + @@ -839,28 +863,37 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { HIVE_ORC_FILE_MEMORY_POOL("hive.exec.orc.memory.pool", 0.5f, "Maximum fraction of heap that can be used by ORC file writers"), HIVE_ORC_WRITE_FORMAT("hive.exec.orc.write.format", null, - "Define the version of the file to write"), + "Define the version of the file to write. Possible values are 0.11 and 0.12.\n" + + "If this parameter is not defined, ORC will use the run length encoding (RLE)\n" + + "introduced in Hive 0.12. Any value other than 0.11 results in the 0.12 encoding."), HIVE_ORC_DEFAULT_STRIPE_SIZE("hive.exec.orc.default.stripe.size", 64L * 1024 * 1024, - "Define the default ORC stripe size"), + "Define the default ORC stripe size, in bytes."), HIVE_ORC_DEFAULT_BLOCK_SIZE("hive.exec.orc.default.block.size", 256L * 1024 * 1024, "Define the default file system block size for ORC files."), HIVE_ORC_DICTIONARY_KEY_SIZE_THRESHOLD("hive.exec.orc.dictionary.key.size.threshold", 0.8f, "If the number of keys in a dictionary is greater than this fraction of the total number of\n" + "non-null rows, turn off dictionary encoding. Use 1 to always use dictionary encoding."), - HIVE_ORC_DEFAULT_ROW_INDEX_STRIDE("hive.exec.orc.default.row.index.stride", 10000, "Define the default ORC index stride"), + HIVE_ORC_DEFAULT_ROW_INDEX_STRIDE("hive.exec.orc.default.row.index.stride", 10000, + "Define the default ORC index stride in number of rows. (Stride is the number of rows\n" + + "an index entry represents.)"), HIVE_ORC_ROW_INDEX_STRIDE_DICTIONARY_CHECK("hive.orc.row.index.stride.dictionary.check", true, "If enabled dictionary check will happen after first row index stride (default 10000 rows)\n" + "else dictionary check will happen before writing first stripe. In both cases, the decision\n" + "to use dictionary or not will be retained thereafter."), - HIVE_ORC_DEFAULT_BUFFER_SIZE("hive.exec.orc.default.buffer.size", 256 * 1024, "Define the default ORC buffer size"), - HIVE_ORC_DEFAULT_BLOCK_PADDING("hive.exec.orc.default.block.padding", true, "Define the default block padding"), + HIVE_ORC_DEFAULT_BUFFER_SIZE("hive.exec.orc.default.buffer.size", 256 * 1024, + "Define the default ORC buffer size, in bytes."), + HIVE_ORC_DEFAULT_BLOCK_PADDING("hive.exec.orc.default.block.padding", true, + "Define the default block padding, which pads stripes to the HDFS block boundaries."), HIVE_ORC_BLOCK_PADDING_TOLERANCE("hive.exec.orc.block.padding.tolerance", 0.05f, - "Define the tolerance for block padding as a percentage of stripe size.\n" + - "For the defaults of 64Mb ORC stripe and 256Mb HDFS blocks, a maximum of 3.2Mb will be reserved for padding within the 256Mb block. \n" + - "In that case, if the available size within the block is more than 3.2Mb, a new smaller stripe will be inserted to fit within that space. \n" + - "This will make sure that no stripe written will cross block boundaries and cause remote reads within a node local task."), + "Define the tolerance for block padding as a decimal fraction of stripe size (for\n" + + "example, the default value 0.05 is 5% of the stripe size). For the defaults of 64Mb\n" + + "ORC stripe and 256Mb HDFS blocks, the default block padding tolerance of 5% will\n" + + "reserve a maximum of 3.2Mb for padding within the 256Mb block. In that case, if the\n" + + "available size within the block is more than 3.2Mb, a new smaller stripe will be\n" + + "inserted to fit within that space. This will make sure that no stripe written will\n" + + "cross block boundaries and cause remote reads within a node local task."), HIVE_ORC_DEFAULT_COMPRESS("hive.exec.orc.default.compress", "ZLIB", "Define the default compression codec for ORC file"), HIVE_ORC_ENCODING_STRATEGY("hive.exec.orc.encoding.strategy", "SPEED", new StringSet("SPEED", "COMPRESSION"), @@ -883,7 +916,8 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { "If ORC reader encounters corrupt data, this value will be used to determine\n" + "whether to skip the corrupt data or throw exception. The default behavior is to throw exception."), - HIVE_ORC_ZEROCOPY("hive.exec.orc.zerocopy", false, "Use zerocopy reads with ORC."), + HIVE_ORC_ZEROCOPY("hive.exec.orc.zerocopy", false, + "Use zerocopy reads with ORC. (This requires Hadoop 2.3 or later.)"), HIVE_LAZYSIMPLE_EXTENDED_BOOLEAN_LITERAL("hive.lazysimple.extended_boolean_literal", false, "LazySimpleSerde uses this property to determine if it treats 'T', 't', 'F', 'f',\n" + @@ -968,6 +1002,9 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { HIVETEZCONTAINERSIZE("hive.tez.container.size", -1, "By default Tez will spawn containers of the size of a mapper. This can be used to overwrite."), + HIVETEZCPUVCORES("hive.tez.cpu.vcores", -1, + "By default Tez will ask for however many cpus map-reduce is configured to use per container.\n" + + "This can be used to overwrite."), HIVETEZJAVAOPTS("hive.tez.java.opts", null, "By default Tez will use the Java options from map tasks. This can be used to overwrite."), HIVETEZLOGLEVEL("hive.tez.log.level", "INFO", @@ -1117,8 +1154,10 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { HIVESTATSAUTOGATHER("hive.stats.autogather", true, "A flag to gather statistics automatically during the INSERT OVERWRITE command."), HIVESTATSDBCLASS("hive.stats.dbclass", "fs", new PatternSet("jdbc(:.*)", "hbase", "counter", "custom", "fs"), - "The storage that stores temporary Hive statistics. Currently, jdbc, hbase, counter and custom type are supported." - ), // StatsSetupConst.StatDB + "The storage that stores temporary Hive statistics. In filesystem based statistics collection ('fs'), \n" + + "each task writes statistics it has collected in a file on the filesystem, which will be aggregated \n" + + "after the job has finished. Supported values are fs (filesystem), jdbc:database (where database \n" + + "can be derby, mysql, etc.), hbase, counter, and custom as defined in StatsSetupConst.java."), // StatsSetupConst.StatDB HIVESTATSJDBCDRIVER("hive.stats.jdbcdriver", "org.apache.derby.jdbc.EmbeddedDriver", "The JDBC driver for the database that stores temporary Hive statistics."), @@ -1251,13 +1290,16 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { // Zookeeper related configs HIVE_ZOOKEEPER_QUORUM("hive.zookeeper.quorum", "", - "List of ZooKeeper servers to talk to. This is needed for: " + - "1. Read/write locks - when hive.lock.manager is set to " + - "org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager, " + - "2. When HiveServer2 supports service discovery via Zookeeper."), + "List of ZooKeeper servers to talk to. This is needed for: \n" + + "1. Read/write locks - when hive.lock.manager is set to \n" + + "org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager, \n" + + "2. When HiveServer2 supports service discovery via Zookeeper.\n" + + "3. For delegation token storage if zookeeper store is used, if\n" + + "hive.cluster.delegation.token.store.zookeeper.connectString is not set"), + HIVE_ZOOKEEPER_CLIENT_PORT("hive.zookeeper.client.port", "2181", - "The port of ZooKeeper servers to talk to. " + - "If the list of Zookeeper servers specified in hive.zookeeper.quorum," + + "The port of ZooKeeper servers to talk to.\n" + + "If the list of Zookeeper servers specified in hive.zookeeper.quorum\n" + "does not contain port numbers, this value is used."), HIVE_ZOOKEEPER_SESSION_TIMEOUT("hive.zookeeper.session.timeout", 600*1000, "ZooKeeper client's session timeout. The client is disconnected, and as a result, all locks released, \n" + @@ -1269,42 +1311,72 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { // Transactions HIVE_TXN_MANAGER("hive.txn.manager", - "org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager", ""), + "org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager", + "Set to org.apache.hadoop.hive.ql.lockmgr.DbTxnManager as part of turning on Hive\n" + + "transactions, which also requires appropriate settings for hive.compactor.initiator.on,\n" + + "hive.compactor.worker.threads, hive.support.concurrency (true), hive.enforce.bucketing\n" + + "(true), and hive.exec.dynamic.partition.mode (nonstrict).\n" + + "The default DummyTxnManager replicates pre-Hive-0.13 behavior and provides\n" + + "no transactions."), HIVE_TXN_TIMEOUT("hive.txn.timeout", "300s", new TimeValidator(TimeUnit.SECONDS), "time after which transactions are declared aborted if the client has not sent a heartbeat."), HIVE_TXN_MAX_OPEN_BATCH("hive.txn.max.open.batch", 1000, "Maximum number of transactions that can be fetched in one call to open_txns().\n" + - "Increasing this will decrease the number of delta files created when\n" + - "streaming data into Hive. But it will also increase the number of\n" + - "open transactions at any given time, possibly impacting read performance."), + "This controls how many transactions streaming agents such as Flume or Storm open\n" + + "simultaneously. The streaming agent then writes that number of entries into a single\n" + + "file (per Flume agent or Storm bolt). Thus increasing this value decreases the number\n" + + "of delta files created by streaming agents. But it also increases the number of open\n" + + "transactions that Hive has to track at any given time, which may negatively affect\n" + + "read performance."), HIVE_COMPACTOR_INITIATOR_ON("hive.compactor.initiator.on", false, - "Whether to run the compactor's initiator thread in this metastore instance or not."), + "Whether to run the initiator and cleaner threads on this metastore instance or not.\n" + + "Set this to true on one instance of the Thrift metastore service as part of turning\n" + + "on Hive transactions. For a complete list of parameters required for turning on\n" + + "transactions, see hive.txn.manager."), HIVE_COMPACTOR_WORKER_THREADS("hive.compactor.worker.threads", 0, - "Number of compactor worker threads to run on this metastore instance."), + "How many compactor worker threads to run on this metastore instance. Set this to a\n" + + "positive number on one or more instances of the Thrift metastore service as part of\n" + + "turning on Hive transactions. For a complete list of parameters required for turning\n" + + "on transactions, see hive.txn.manager.\n" + + "Worker threads spawn MapReduce jobs to do compactions. They do not do the compactions\n" + + "themselves. Increasing the number of worker threads will decrease the time it takes\n" + + "tables or partitions to be compacted once they are determined to need compaction.\n" + + "It will also increase the background load on the Hadoop cluster as more MapReduce jobs\n" + + "will be running in the background."), HIVE_COMPACTOR_WORKER_TIMEOUT("hive.compactor.worker.timeout", "86400s", new TimeValidator(TimeUnit.SECONDS), - "Time before a given compaction in working state is declared a failure\n" + - "and returned to the initiated state."), + "Time in seconds after which a compaction job will be declared failed and the\n" + + "compaction re-queued."), HIVE_COMPACTOR_CHECK_INTERVAL("hive.compactor.check.interval", "300s", new TimeValidator(TimeUnit.SECONDS), - "Time between checks to see if any partitions need compacted.\n" + - "This should be kept high because each check for compaction requires many calls against the NameNode."), + "Time in seconds between checks to see if any tables or partitions need to be\n" + + "compacted. This should be kept high because each check for compaction requires\n" + + "many calls against the NameNode.\n" + + "Decreasing this value will reduce the time it takes for compaction to be started\n" + + "for a table or partition that requires compaction. However, checking if compaction\n" + + "is needed requires several calls to the NameNode for each table or partition that\n" + + "has had a transaction done on it since the last major compaction. So decreasing this\n" + + "value will increase the load on the NameNode."), HIVE_COMPACTOR_DELTA_NUM_THRESHOLD("hive.compactor.delta.num.threshold", 10, - "Number of delta files that must exist in a directory before the compactor will attempt\n" + - "a minor compaction."), + "Number of delta directories in a table or partition that will trigger a minor\n" + + "compaction."), HIVE_COMPACTOR_DELTA_PCT_THRESHOLD("hive.compactor.delta.pct.threshold", 0.1f, - "Percentage (by size) of base that deltas can be before major compaction is initiated."), + "Percentage (fractional) size of the delta files relative to the base that will trigger\n" + + "a major compaction. (1.0 = 100%, so the default 0.1 = 10%.)"), HIVE_COMPACTOR_ABORTEDTXN_THRESHOLD("hive.compactor.abortedtxn.threshold", 1000, - "Number of aborted transactions involving a particular table or partition before major\n" + - "compaction is initiated."), + "Number of aborted transactions involving a given table or partition that will trigger\n" + + "a major compaction."), + + HIVE_COMPACTOR_CLEANER_RUN_INTERVAL("hive.compactor.cleaner.run.interval", "5000ms", + new TimeValidator(TimeUnit.MILLISECONDS), "Time between runs of the cleaner thread"), // For HBase storage handler HIVE_HBASE_WAL_ENABLED("hive.hbase.wal.enabled", true, @@ -1324,10 +1396,11 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { HIVEOUTERJOINSUPPORTSFILTERS("hive.outerjoin.supports.filters", true, ""), - HIVEFETCHTASKCONVERSION("hive.fetch.task.conversion", "more", new StringSet("minimal", "more"), + HIVEFETCHTASKCONVERSION("hive.fetch.task.conversion", "more", new StringSet("none", "minimal", "more"), "Some select queries can be converted to single FETCH task minimizing latency.\n" + "Currently the query should be single sourced not having any subquery and should not have\n" + "any aggregations or distincts (which incurs RS), lateral views and joins.\n" + + "0. none : disable hive.fetch.task.conversion\n" + "1. minimal : SELECT STAR, FILTER on partition columns, LIMIT only\n" + "2. more : SELECT, FILTER, LIMIT only (support TABLESAMPLE and virtual columns)" ), @@ -1351,7 +1424,12 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { "The SerDe used by FetchTask to serialize the fetch output."), HIVEEXPREVALUATIONCACHE("hive.cache.expr.evaluation", true, - "If true, evaluation result of deterministic expression referenced twice or more will be cached."), + "If true, the evaluation result of a deterministic expression referenced twice or more\n" + + "will be cached.\n" + + "For example, in a filter condition like '.. where key + 10 = 100 or key + 10 = 0'\n" + + "the expression 'key + 10' will be evaluated/cached once and reused for the following\n" + + "expression ('key + 10 = 0'). Currently, this is applied only to expressions in select\n" + + "or filter operators."), // Hive Variables HIVEVARIABLESUBSTITUTE("hive.variable.substitute", true, @@ -1377,9 +1455,11 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { "interface org.apache.hadoop.hive.ql.security.HiveAuthenticationProvider."), HIVE_METASTORE_AUTHORIZATION_MANAGER("hive.security.metastore.authorization.manager", "org.apache.hadoop.hive.ql.security.authorization.DefaultHiveMetastoreAuthorizationProvider", - "authorization manager class name to be used in the metastore for authorization.\n" + - "The user defined authorization class should implement interface \n" + - "org.apache.hadoop.hive.ql.security.authorization.HiveMetastoreAuthorizationProvider. "), + "Names of authorization manager classes (comma separated) to be used in the metastore\n" + + "for authorization. The user defined authorization class should implement interface\n" + + "org.apache.hadoop.hive.ql.security.authorization.HiveMetastoreAuthorizationProvider.\n" + + "All authorization manager classes have to successfully authorize the metastore API\n" + + "call for the command execution to be allowed."), HIVE_METASTORE_AUTHORIZATION_AUTH_READS("hive.security.metastore.authorization.auth.reads", true, "If this is true, metastore authorizer authorizes read actions on database, table"), HIVE_METASTORE_AUTHENTICATOR_MANAGER("hive.security.metastore.authenticator.manager", @@ -1399,15 +1479,30 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { "the privileges automatically granted to some roles whenever a table gets created.\n" + "An example like \"roleX,roleY:select;roleZ:create\" will grant select privilege to roleX and roleY,\n" + "and grant create privilege to roleZ whenever a new table created."), - HIVE_AUTHORIZATION_TABLE_OWNER_GRANTS("hive.security.authorization.createtable.owner.grants", "", - "the privileges automatically granted to the owner whenever a table gets created.\n" + - "An example like \"select,drop\" will grant select and drop privilege to the owner of the table"), + HIVE_AUTHORIZATION_TABLE_OWNER_GRANTS("hive.security.authorization.createtable.owner.grants", + "", + "The privileges automatically granted to the owner whenever a table gets created.\n" + + "An example like \"select,drop\" will grant select and drop privilege to the owner\n" + + "of the table. Note that the default gives the creator of a table no access to the\n" + + "table (but see HIVE-8067)."), - // if this is not set default value is added by sql standard authorizer. + // if this is not set default value is set during config initialization // Default value can't be set in this constructor as it would refer names in other ConfVars // whose constructor would not have been called - HIVE_AUTHORIZATION_SQL_STD_AUTH_CONFIG_WHITELIST("hive.security.authorization.sqlstd.confwhitelist", "", - "interal variable. List of modifiable configurations by user."), + HIVE_AUTHORIZATION_SQL_STD_AUTH_CONFIG_WHITELIST( + "hive.security.authorization.sqlstd.confwhitelist", "", + "List of comma separated Java regexes. Configurations parameters that match these\n" + + "regexes can be modified by user when SQL standard authorization is enabled.\n" + + "To get the default value, use the 'set ' command.\n" + + "Note that the hive.conf.restricted.list checks are still enforced after the white list\n" + + "check"), + + HIVE_AUTHORIZATION_SQL_STD_AUTH_CONFIG_WHITELIST_APPEND( + "hive.security.authorization.sqlstd.confwhitelist.append", "", + "List of comma separated Java regexes, to be appended to list set in\n" + + "hive.security.authorization.sqlstd.confwhitelist. Using this list instead\n" + + "of updating the original list means that you can append to the defaults\n" + + "set by SQL standard authorization instead of replacing it entirely."), HIVE_CLI_PRINT_HEADER("hive.cli.print.header", false, "Whether to print the names of the columns in query output."), @@ -1460,6 +1555,10 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { "If the property is set, the value must be a valid URI (java.net.URI, e.g. \"file:///tmp/my-logging.properties\"), \n" + "which you can then extract a URL from and pass to PropertyConfigurator.configure(URL)."), + HIVE_LOG_EXPLAIN_OUTPUT("hive.log.explain.output", false, + "Whether to log explain output for every query.\n" + + "When enabled, will log EXPLAIN EXTENDED output for the query at INFO log4j log level."), + // prefix used to auto generated column aliases (this should be started with '_') HIVE_AUTOGEN_COLUMNALIAS_PREFIX_LABEL("hive.autogen.columnalias.prefix.label", "_c", "String used as a prefix when auto generating column alias.\n" + @@ -1538,12 +1637,13 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { // binary transport settings HIVE_SERVER2_THRIFT_PORT("hive.server2.thrift.port", 10000, "Port number of HiveServer2 Thrift interface when hive.server2.transport.mode is 'binary'."), - // hadoop.rpc.protection being set to a higher level than HiveServer2 - // does not make sense in most situations. - // HiveServer2 ignores hadoop.rpc.protection in favor of hive.server2.thrift.sasl.qop. - HIVE_SERVER2_THRIFT_SASL_QOP("hive.server2.thrift.sasl.qop", "auth", new StringSet("auth", "auth-int", "auth-conf"), - "Sasl QOP value; Set it to one of following values to enable higher levels of\n" + - " protection for HiveServer2 communication with clients.\n" + + HIVE_SERVER2_THRIFT_SASL_QOP("hive.server2.thrift.sasl.qop", "auth", + new StringSet("auth", "auth-int", "auth-conf"), + "Sasl QOP value; set it to one of following values to enable higher levels of\n" + + "protection for HiveServer2 communication with clients.\n" + + "Setting hadoop.rpc.protection to a higher level than HiveServer2 does not\n" + + "make sense in most situations. HiveServer2 ignores hadoop.rpc.protection in favor\n" + + "of hive.server2.thrift.sasl.qop.\n" + " \"auth\" - authentication only (default)\n" + " \"auth-int\" - authentication plus integrity protection\n" + " \"auth-conf\" - authentication plus integrity and confidentiality protection\n" + @@ -1561,7 +1661,7 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { "Number of threads in the async thread pool for HiveServer2"), HIVE_SERVER2_ASYNC_EXEC_SHUTDOWN_TIMEOUT("hive.server2.async.exec.shutdown.timeout", "10s", new TimeValidator(TimeUnit.SECONDS), - "Maximum time for which HiveServer2 shutdown will wait for async"), + "How long HiveServer2 shutdown will wait for async threads to terminate."), HIVE_SERVER2_ASYNC_EXEC_WAIT_QUEUE_SIZE("hive.server2.async.exec.wait.queue.size", 100, "Size of the wait queue for async thread pool in HiveServer2.\n" + "After hitting this limit, the async thread pool will reject new requests."), @@ -1581,7 +1681,9 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { " LDAP: LDAP/AD based authentication\n" + " KERBEROS: Kerberos/GSSAPI authentication\n" + " CUSTOM: Custom authentication provider\n" + - " (Use with property hive.server2.custom.authentication.class)"), + " (Use with property hive.server2.custom.authentication.class)\n" + + " PAM: Pluggable authentication module\n" + + " NOSASL: Raw transport"), HIVE_SERVER2_ALLOW_USER_SUBSTITUTION("hive.server2.allow.user.substitution", true, "Allow alternate user to be specified as part of HiveServer2 open connection request."), HIVE_SERVER2_KERBEROS_KEYTAB("hive.server2.authentication.kerberos.keytab", "", @@ -1629,9 +1731,18 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { " HIVE : Exposes Hive's native table types like MANAGED_TABLE, EXTERNAL_TABLE, VIRTUAL_VIEW\n" + " CLASSIC : More generic types like TABLE and VIEW"), HIVE_SERVER2_SESSION_HOOK("hive.server2.session.hook", "", ""), - HIVE_SERVER2_USE_SSL("hive.server2.use.SSL", false, ""), - HIVE_SERVER2_SSL_KEYSTORE_PATH("hive.server2.keystore.path", "", ""), - HIVE_SERVER2_SSL_KEYSTORE_PASSWORD("hive.server2.keystore.password", "", ""), + HIVE_SERVER2_USE_SSL("hive.server2.use.SSL", false, + "Set this to true for using SSL encryption in HiveServer2."), + HIVE_SERVER2_SSL_KEYSTORE_PATH("hive.server2.keystore.path", "", + "SSL certificate keystore location."), + HIVE_SERVER2_SSL_KEYSTORE_PASSWORD("hive.server2.keystore.password", "", + "SSL certificate keystore password."), + HIVE_SERVER2_MAP_FAIR_SCHEDULER_QUEUE("hive.server2.map.fair.scheduler.queue", true, + "If the YARN fair scheduler is configured and HiveServer2 is running in non-impersonation mode,\n" + + "this setting determines the user for fair scheduler queue mapping.\n" + + "If set to true (default), the logged-in user determines the fair scheduler queue\n" + + "for submitted jobs, so that map reduce resource usage can be tracked by user.\n" + + "If set to false, all Hive jobs go to the 'hive' user's queue."), HIVE_SECURITY_COMMAND_WHITELIST("hive.security.command.whitelist", "set,reset,dfs,add,list,delete,reload,compile", "Comma separated list of non-SQL Hive commands users are authorized to execute"), @@ -1731,8 +1842,10 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { HIVE_VECTORIZATION_GROUPBY_FLUSH_PERCENT("hive.vectorized.groupby.flush.percent", (float) 0.1, "Percent of entries in the group by aggregation hash flushed when the memory threshold is exceeded."), - HIVE_TYPE_CHECK_ON_INSERT("hive.typecheck.on.insert", true, ""), + HIVE_HADOOP_CLASSPATH("hive.hadoop.classpath", null, + "For Windows OS, we need to pass HIVE_HADOOP_CLASSPATH Java parameter while starting HiveServer2 \n" + + "using \"-hiveconf hive.hadoop.classpath=%HIVE_LIB%\"."), HIVE_RPC_QUERY_PLAN("hive.rpc.query.plan", false, "Whether to send the query plan via local resource or RPC"), @@ -1799,12 +1912,17 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { "of reducers that tez specifies."), TEZ_DYNAMIC_PARTITION_PRUNING( "hive.tez.dynamic.partition.pruning", true, - "When dynamic pruning is enabled, joins on partition keys will be processed by sending events from the processing " + - "vertices to the tez application master. These events will be used to prune unnecessary partitions."), + "When dynamic pruning is enabled, joins on partition keys will be processed by sending\n" + + "events from the processing vertices to the Tez application master. These events will be\n" + + "used to prune unnecessary partitions."), TEZ_DYNAMIC_PARTITION_PRUNING_MAX_EVENT_SIZE("hive.tez.dynamic.partition.pruning.max.event.size", 1*1024*1024L, "Maximum size of events sent by processors in dynamic pruning. If this size is crossed no pruning will take place."), TEZ_DYNAMIC_PARTITION_PRUNING_MAX_DATA_SIZE("hive.tez.dynamic.partition.pruning.max.data.size", 100*1024*1024L, - "Maximum total data size of events in dynamic pruning.") + "Maximum total data size of events in dynamic pruning."), + TEZ_SMB_NUMBER_WAVES( + "hive.tez.smb.number.waves", + (float) 0.5, + "The number of waves in which to run the SMB join. Account for cluster being occupied. Ideally should be 1 wave.") ; public final String varname; @@ -2030,8 +2148,9 @@ private static synchronized InputStream getConfVarInputStream() { } public void verifyAndSet(String name, String value) throws IllegalArgumentException { - if (isWhiteListRestrictionEnabled) { - if (!modWhiteList.contains(name)) { + if (modWhiteListPattern != null) { + Matcher wlMatcher = modWhiteListPattern.matcher(name); + if (!wlMatcher.matches()) { throw new IllegalArgumentException("Cannot modify " + name + " at runtime. " + "It is not in list of params that are allowed to be modified at runtime"); } @@ -2102,7 +2221,7 @@ public static long toTime(String value, TimeUnit inputUnit, TimeUnit outUnit) { public static TimeUnit unitFor(String unit, TimeUnit defaultUnit) { unit = unit.trim().toLowerCase(); - if (unit.isEmpty()) { + if (unit.isEmpty() || unit.equals("l")) { if (defaultUnit == null) { throw new IllegalArgumentException("Time unit is not specified"); } @@ -2369,11 +2488,146 @@ private void initialize(Class cls) { unset(key); } } + + setupSQLStdAuthWhiteList(); + // setup list of conf vars that are not allowed to change runtime setupRestrictList(); + + } + + /** + * If the config whitelist param for sql standard authorization is not set, set it up here. + */ + private void setupSQLStdAuthWhiteList() { + String whiteListParamsStr = getVar(ConfVars.HIVE_AUTHORIZATION_SQL_STD_AUTH_CONFIG_WHITELIST); + if (whiteListParamsStr == null || whiteListParamsStr.trim().isEmpty()) { + // set the default configs in whitelist + whiteListParamsStr = getSQLStdAuthDefaultWhiteListPattern(); + } + setVar(ConfVars.HIVE_AUTHORIZATION_SQL_STD_AUTH_CONFIG_WHITELIST, whiteListParamsStr); + } + + private static String getSQLStdAuthDefaultWhiteListPattern() { + // create the default white list from list of safe config params + // and regex list + String confVarPatternStr = Joiner.on("|").join(convertVarsToRegex(sqlStdAuthSafeVarNames)); + String regexPatternStr = Joiner.on("|").join(sqlStdAuthSafeVarNameRegexes); + return regexPatternStr + "|" + confVarPatternStr; + } + + /** + * @param paramList list of parameter strings + * @return list of parameter strings with "." replaced by "\." + */ + private static String[] convertVarsToRegex(String[] paramList) { + String[] regexes = new String[paramList.length]; + for(int i=0; i org.apache.hive hive - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../pom.xml diff --git a/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/MultiDelimitSerDe.java b/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/MultiDelimitSerDe.java index b61ad33..72f4234 100644 --- a/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/MultiDelimitSerDe.java +++ b/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/MultiDelimitSerDe.java @@ -55,6 +55,14 @@ * Currently field.delim can be multiple character while collection.delim * and mapkey.delim should be just single character. */ +@SerDeSpec(schemaProps = { + serdeConstants.LIST_COLUMNS, serdeConstants.LIST_COLUMN_TYPES, + serdeConstants.FIELD_DELIM, serdeConstants.COLLECTION_DELIM, serdeConstants.MAPKEY_DELIM, + serdeConstants.SERIALIZATION_FORMAT, serdeConstants.SERIALIZATION_NULL_FORMAT, + serdeConstants.SERIALIZATION_LAST_COLUMN_TAKES_REST, + serdeConstants.ESCAPE_CHAR, + serdeConstants.SERIALIZATION_ENCODING, + LazySimpleSerDe.SERIALIZATION_EXTEND_NESTING_LEVELS}) public class MultiDelimitSerDe extends AbstractSerDe { private static final Log LOG = LogFactory.getLog(MultiDelimitSerDe.class.getName()); private static final byte[] DEFAULT_SEPARATORS = {(byte) 1, (byte) 2, (byte) 3}; diff --git a/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/RegexSerDe.java b/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/RegexSerDe.java index f44b4a5..aadfb51 100644 --- a/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/RegexSerDe.java +++ b/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/RegexSerDe.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.AbstractSerDe; import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.SerDeSpec; import org.apache.hadoop.hive.serde2.SerDeStats; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; @@ -70,10 +71,18 @@ * writableStringObjectInspector. We should switch to that when we have a UTF-8 * based Regex library. */ +@SerDeSpec(schemaProps = { + serdeConstants.LIST_COLUMNS, serdeConstants.LIST_COLUMN_TYPES, + RegexSerDe.INPUT_REGEX, RegexSerDe.OUTPUT_FORMAT_STRING, + RegexSerDe.INPUT_REGEX_CASE_SENSITIVE }) public class RegexSerDe extends AbstractSerDe { public static final Log LOG = LogFactory.getLog(RegexSerDe.class.getName()); + public static final String INPUT_REGEX = "input.regex"; + public static final String OUTPUT_FORMAT_STRING = "output.format.string"; + public static final String INPUT_REGEX_CASE_SENSITIVE = "input.regex.case.insensitive"; + int numColumns; String inputRegex; String outputFormatString; @@ -90,12 +99,12 @@ public void initialize(Configuration conf, Properties tbl) // We can get the table definition from tbl. // Read the configuration parameters - inputRegex = tbl.getProperty("input.regex"); - outputFormatString = tbl.getProperty("output.format.string"); + inputRegex = tbl.getProperty(INPUT_REGEX); + outputFormatString = tbl.getProperty(OUTPUT_FORMAT_STRING); String columnNameProperty = tbl.getProperty(serdeConstants.LIST_COLUMNS); String columnTypeProperty = tbl.getProperty(serdeConstants.LIST_COLUMN_TYPES); boolean inputRegexIgnoreCase = "true".equalsIgnoreCase(tbl - .getProperty("input.regex.case.insensitive")); + .getProperty(INPUT_REGEX_CASE_SENSITIVE)); // Parse the configuration parameters if (inputRegex != null) { @@ -258,6 +267,7 @@ public Writable serialize(Object obj, ObjectInspector objInspector) return outputRowText; } + @Override public SerDeStats getSerDeStats() { // no support for statistics return null; diff --git a/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/TypedBytesSerDe.java b/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/TypedBytesSerDe.java index f79f821..ea87bf6 100644 --- a/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/TypedBytesSerDe.java +++ b/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/TypedBytesSerDe.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.AbstractSerDe; import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.SerDeSpec; import org.apache.hadoop.hive.serde2.SerDeStats; import org.apache.hadoop.hive.serde2.SerDeUtils; import org.apache.hadoop.hive.serde2.io.ByteWritable; @@ -73,6 +74,7 @@ * this, which is apparently 25% faster than the python version is available at * http://github.com/klbostee/ctypedbytes/tree/master */ +@SerDeSpec(schemaProps = {serdeConstants.LIST_COLUMNS, serdeConstants.LIST_COLUMN_TYPES}) public class TypedBytesSerDe extends AbstractSerDe { public static final Log LOG = LogFactory.getLog(TypedBytesSerDe.class diff --git a/data/conf/hive-log4j.properties b/data/conf/hive-log4j.properties index 7f5dfc4..f90b833 100644 --- a/data/conf/hive-log4j.properties +++ b/data/conf/hive-log4j.properties @@ -75,6 +75,11 @@ log4j.category.JPOX.Query=ERROR,DRFA log4j.category.JPOX.General=ERROR,DRFA log4j.category.JPOX.Enhancer=ERROR,DRFA log4j.logger.org.apache.hadoop.conf.Configuration=ERROR,DRFA +log4j.logger.org.apache.zookeeper=INFO,DRFA +log4j.logger.org.apache.zookeeper.server.ServerCnxn=WARN,DRFA log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN,DRFA +log4j.logger.org.apache.zookeeper.ClientCnxn=WARN,DRFA +log4j.logger.org.apache.zookeeper.ClientCnxnSocket=WARN,DRFA log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN,DRFA - +log4j.logger.org.apache.hadoop.hive.ql.log.PerfLogger=WARN,DRFA +log4j.logger.org.apache.hadoop.hive.ql.exec.Operator=INFO,DRFA diff --git a/data/files/avro_date.txt b/data/files/avro_date.txt new file mode 100644 index 0000000..0858896 --- /dev/null +++ b/data/files/avro_date.txt @@ -0,0 +1,4 @@ +2012-02-21|foo:1980-12-16,bar:1998-05-07|2011-09-04,2011-09-05 +2014-02-11|baz:1981-12-16|2011-09-05 +1947-02-11|baz:1921-12-16|2011-09-05 +8200-02-11|baz:6981-12-16|1039-09-05 diff --git a/data/files/customer_address.txt b/data/files/customer_address.txt new file mode 100644 index 0000000..81e05af --- /dev/null +++ b/data/files/customer_address.txt @@ -0,0 +1,20 @@ +1|AAAAAAAABAAAAAAA|18|Jackson |Parkway|Suite 280|Fairfield|Maricopa County|AZ|86192|United States|-7|condo| +2|AAAAAAAACAAAAAAA|362|Washington 6th|RD|Suite 80|Fairview|Taos County|NM|85709|United States|-7|condo| +3|AAAAAAAADAAAAAAA|585|Dogwood Washington|Circle|Suite Q|Pleasant Valley|York County|PA|12477|United States|-5|single family| +4|AAAAAAAAEAAAAAAA|111|Smith |Wy|Suite A|Oak Ridge|Kit Carson County|CO|88371|United States|-7|condo| +5|AAAAAAAAFAAAAAAA|31|College |Blvd|Suite 180|Glendale|Barry County|MO|63951|United States|-6|single family| +6|AAAAAAAAGAAAAAAA|59|Williams Sixth|Parkway|Suite 100|Lakeview|Chelan County|WA|98579|United States|-8|single family| +7|AAAAAAAAHAAAAAAA||Hill 7th|Road|Suite U|Farmington|||39145|United States||| +8|AAAAAAAAIAAAAAAA|875|Lincoln |Ct.|Suite Y|Union|Bledsoe County|TN|38721|United States|-5|apartment| +9|AAAAAAAAJAAAAAAA|819|1st Laurel|Ave|Suite 70|New Hope|Perry County|AL|39431|United States|-6|condo| +10|AAAAAAAAKAAAAAAA|851|Woodland Poplar|ST|Suite Y|Martinsville|Haines Borough|AK|90419|United States|-9|condo| +11|AAAAAAAALAAAAAAA|189|13th 2nd|Street|Suite 470|Maple Grove|Madison County|MT|68252|United States|-7|single family| +12|AAAAAAAAMAAAAAAA|76|Ash 8th|Ct.|Suite O|Edgewood|Mifflin County|PA|10069|United States|-5|apartment| +13|AAAAAAAANAAAAAAA|424|Main Second|Ln|Suite 130|Greenville|Noxubee County|MS|51387|United States|-6|single family| +14|AAAAAAAAOAAAAAAA|923|Pine Oak|Dr.|Suite 100||Lipscomb County|TX|77752||-6|| +15|AAAAAAAAPAAAAAAA|314|Spring |Ct.|Suite B|Oakland|Washington County|OH|49843|United States|-5|apartment| +16|AAAAAAAAABAAAAAA|576|Adams Center|Street|Suite J|Valley View|Oldham County|TX|75124|United States|-6|condo| +17|AAAAAAAABBAAAAAA|801|Green |Dr.|Suite 0|Montpelier|Richland County|OH|48930|United States|-5|single family| +18|AAAAAAAACBAAAAAA|460|Maple Spruce|Court|Suite 480|Somerville|Potter County|SD|57783|United States|-7|condo| +19|AAAAAAAADBAAAAAA|611|Wilson |Way|Suite O|Oakdale|Tangipahoa Parish|LA|79584|United States|-6|apartment| +20|AAAAAAAAEBAAAAAA|675|Elm Wilson|Street|Suite I|Hopewell|Williams County|OH|40587|United States|-5|condo| diff --git a/data/files/map_null_schema.avro b/data/files/map_null_schema.avro new file mode 100644 index 0000000..d8ceaa7 --- /dev/null +++ b/data/files/map_null_schema.avro @@ -0,0 +1 @@ +{"type":"record","name":"dku_record_0","namespace":"com.dataiku.dss","doc":"","fields":[{"name":"avreau_col_1","type":["null",{"type":"map","values":["null","string"]}],"default":null}]} diff --git a/data/files/store.txt b/data/files/store.txt new file mode 100644 index 0000000..078bafd --- /dev/null +++ b/data/files/store.txt @@ -0,0 +1,12 @@ +1|AAAAAAAABAAAAAAA|1997-03-13||2451189|ought|245|5250760|8AM-4PM|William Ward|2|Unknown|Enough high areas stop expectations. Elaborate, local is|Charles Bartley|1|Unknown|1|Unknown|767|Spring |Wy|Suite 250|Midway|Williamson County|TN|31904|United States|-5|0.03| +2|AAAAAAAACAAAAAAA|1997-03-13|2000-03-12||able|236|5285950|8AM-4PM|Scott Smith|8|Unknown|Parliamentary candidates wait then heavy, keen mil|David Lamontagne|1|Unknown|1|Unknown|255|Sycamore |Dr.|Suite 410|Midway|Williamson County|TN|31904|United States|-5|0.03| +3|AAAAAAAACAAAAAAA|2000-03-13|||able|236|7557959|8AM-4PM|Scott Smith|7|Unknown|Impossible, true arms can treat constant, complete w|David Lamontagne|1|Unknown|1|Unknown|877|Park Laurel|Road|Suite T|Midway|Williamson County|TN|31904|United States|-5|0.03| +4|AAAAAAAAEAAAAAAA|1997-03-13|1999-03-13|2451044|ese|218|9341467|8AM-4PM|Edwin Adams|4|Unknown|Events would achieve other, eastern hours. Mechanisms must not eat other, new org|Thomas Pollack|1|Unknown|1|Unknown|27|Lake |Ln|Suite 260|Midway|Williamson County|TN|31904|United States|-5|0.03| +5|AAAAAAAAEAAAAAAA|1999-03-14|2001-03-12|2450910|anti|288|9078805|8AM-4PM|Edwin Adams|8|Unknown|Events would achieve other, eastern hours. Mechanisms must not eat other, new org|Thomas Pollack|1|Unknown|1|Unknown|27|Lee 6th|Court|Suite 80|Fairview|Williamson County|TN|35709|United States|-5|0.03| +6|AAAAAAAAEAAAAAAA|2001-03-13|||cally|229|9026222|8AM-4PM|Edwin Adams|10|Unknown|Events would achieve other, eastern hours. Mechanisms must not eat other, new org|Thomas Pollack|1|Unknown|1|Unknown|220|6th |Lane|Suite 140|Midway|Williamson County|TN|31904|United States|-5|0.03| +7|AAAAAAAAHAAAAAAA|1997-03-13|||ation|297|8954883|8AM-4PM|David Thomas|9|Unknown|Architects coul|Thomas Benton|1|Unknown|1|Unknown|811|Lee |Circle|Suite T|Midway|Williamson County|TN|31904|United States|-5|0.01| +8|AAAAAAAAIAAAAAAA|1997-03-13|2000-03-12||eing|278|6995995|8AM-4PM|Brett Yates|2|Unknown|Various bars make most. Difficult levels introduce at a boots. Buildings welcome only never el|Dean Morrison|1|Unknown|1|Unknown|226|12th |Lane|Suite D|Fairview|Williamson County|TN|35709|United States|-5|0.08| +9|AAAAAAAAIAAAAAAA|2000-03-13|||eing|271|6995995|8AM-4PM|Brett Yates|2|Unknown|Formal, psychological pounds relate reasonable, young principles. Black, |Dean Morrison|1|Unknown|1|Unknown|226|Hill |Boulevard|Suite 190|Midway|Williamson County|TN|31904|United States|-5|0.08| +10|AAAAAAAAKAAAAAAA|1997-03-13|1999-03-13||bar|294|9294113|8AM-4PM|Raymond Jacobs|8|Unknown|Little expectations include yet forward meetings.|Michael Wilson|1|Unknown|1|Unknown|175|4th |Court|Suite C|Midway|Williamson County|TN|31904|United States|-5|0.06| +11|AAAAAAAAKAAAAAAA|1999-03-14|2001-03-12||ought|294|9294113|8AM-4PM|Raymond Jacobs|6|Unknown|Mysterious employe|Michael Wilson|1|Unknown|1|Unknown|175|Park Green|Court|Suite 160|Midway|Williamson County|TN|31904|United States|-5|0.11| +12|AAAAAAAAKAAAAAAA|2001-03-13|||ought|294|5219562|8AM-12AM|Robert Thompson|6|Unknown|Events develop i|Dustin Kelly|1|Unknown|1|Unknown|337|College |Boulevard|Suite 100|Fairview|Williamson County|TN|31904|United States|-5|0.01| diff --git a/data/files/store_sales.txt b/data/files/store_sales.txt new file mode 100644 index 0000000..a39881c --- /dev/null +++ b/data/files/store_sales.txt @@ -0,0 +1,1000 @@ +2451813|65495|3617|67006|591617|3428|24839|10|161|1|79|11.41|18.71|2.80|99.54|221.20|901.39|1478.09|6.08|99.54|121.66|127.74|-779.73| +2451813|65495|13283|67006|591617|3428|24839|10|154|1|37|63.63|101.17|41.47|46.03|1534.39|2354.31|3743.29|59.53|46.03|1488.36|1547.89|-865.95| +2451813|65495|13631|67006|591617|3428|24839|10|172|1|99|80.52|137.68|83.98|0.00|8314.02|7971.48|13630.32|0.00|0.00|8314.02|8314.02|342.54| +2451813|65495|5981|67006|591617|3428|24839|10|280|1|14|57.37|76.30|6.10|0.00|85.40|803.18|1068.20|0.00|0.00|85.40|85.40|-717.78| +2451813|65495|4553|67006|591617|3428|24839|10|236|1|100|25.08|36.86|0.73|0.00|73.00|2508.00|3686.00|6.57|0.00|73.00|79.57|-2435.00| +2451813|65495|10993|67006|591617|3428|24839|10|263|1|91|93.48|108.43|93.24|0.00|8484.84|8506.68|9867.13|254.54|0.00|8484.84|8739.38|-21.84| +2451813|65495|49|67006|591617|3428|24839|10|70|1|5|10.68|15.91|6.68|0.00|33.40|53.40|79.55|2.33|0.00|33.40|35.73|-20.00| +2451813|65495|4583|67006|591617|3428|24839|10|267|1|72|84.72|111.83|61.50|0.00|4428.00|6099.84|8051.76|177.12|0.00|4428.00|4605.12|-1671.84| +2451813|65495|13538|67006|591617|3428|24839|10|106|1|14|11.54|11.77|0.00|0.00|0.00|161.56|164.78|0.00|0.00|0.00|0.00|-161.56| +2451813|65495|3248|67006|591617|3428|24839|10|189|1|58|4.57|5.34|3.52|0.00|204.16|265.06|309.72|0.00|0.00|204.16|204.16|-60.90| +2451813|65495|4682|67006|591617|3428|24839|10|7|1|44|31.07|54.99|24.19|0.00|1064.36|1367.08|2419.56|85.14|0.00|1064.36|1149.50|-302.72| +2451813|65495|5527|67006|591617|3428|24839|10|112|1|88|52.41|63.94|49.23|0.00|4332.24|4612.08|5626.72|259.93|0.00|4332.24|4592.17|-279.84| +2451813|65495|1553|67006|591617|3428|24839|10|242|1|50|67.71|126.61|87.36|0.00|4368.00|3385.50|6330.50|349.44|0.00|4368.00|4717.44|982.50| +2451813|65495|173|67006|591617|3428|24839|10|249|1|65|27.16|52.14|41.19|0.00|2677.35|1765.40|3389.10|214.18|0.00|2677.35|2891.53|911.95| +2451524|75316|6928|68284|1712478|4672|13266|10|37|2|65|93.86|158.62|34.89|0.00|2267.85|6100.90|10310.30|45.35|0.00|2267.85|2313.20|-3833.05| +2451524|75316|13927|68284|1712478|4672|13266|10|131|2|93|35.87|49.14|15.23|0.00|1416.39|3335.91|4570.02|113.31|0.00|1416.39|1529.70|-1919.52| +2451524|75316|10768|68284|1712478|4672|13266|10|78|2|30|2.27|3.83|1.11|0.00|33.30|68.10|114.90|0.00|0.00|33.30|33.30|-34.80| +2451524|75316|14701|68284|1712478|4672|13266|10|260|2|58|53.09|58.39|7.59|0.00|440.22|3079.22|3386.62|0.00|0.00|440.22|440.22|-2639.00| +|75316|17420|||4672||10||2||17.12|19.85||||||19.07|||495.84|| +2451524|75316|9436|68284|1712478|4672|13266|10|144|2|79|88.02|128.50|69.39|4549.90|5481.81|6953.58|10151.50|83.87|4549.90|931.91|1015.78|-6021.67| +2451524|75316|7654|68284|1712478|4672|13266|10|111|2|25|74.26|89.11|35.64|0.00|891.00|1856.50|2227.75|8.91|0.00|891.00|899.91|-965.50| +2451524|75316|12223|68284|1712478|4672|13266|10|267|2|78|65.71|88.70|23.06|341.74|1798.68|5125.38|6918.60|131.12|341.74|1456.94|1588.06|-3668.44| +2451524|75316|15085|68284|1712478|4672|13266|10|107|2|88|64.43|79.89|41.54|0.00|3655.52|5669.84|7030.32|146.22|0.00|3655.52|3801.74|-2014.32| +2451524|75316|3740|68284|1712478|4672|13266|10|72|2|49|6.55|7.86|7.46|0.00|365.54|320.95|385.14|0.00|0.00|365.54|365.54|44.59| +2451524|75316|1363|68284|1712478|4672|13266|10|38|2|4|13.46|15.34|12.57|0.00|50.28|53.84|61.36|4.02|0.00|50.28|54.30|-3.56| +2451524|75316|12068|68284|1712478|4672|13266|10|102|2|74|16.55|17.21|4.47|0.00|330.78|1224.70|1273.54|0.00|0.00|330.78|330.78|-893.92| +2451524|75316|13340|68284|1712478|4672|13266|10|212|2|71|36.01|64.09|3.20|0.00|227.20|2556.71|4550.39|4.54|0.00|227.20|231.74|-2329.51| +2451524|75316|2930|68284|1712478|4672|13266|10|234|2|36|61.23|93.06|11.16|0.00|401.76|2204.28|3350.16|28.12|0.00|401.76|429.88|-1802.52| +2451524|75316|15782|68284|1712478|4672|13266|10|268|2|62|77.97|85.76|79.75|0.00|4944.50|4834.14|5317.12|0.00|0.00|4944.50|4944.50|110.36| +2452638|50412|12447|44047|586372|697|22622|1|293|3|82|56.14|111.15|46.68|0.00|3827.76|4603.48|9114.30|306.22|0.00|3827.76|4133.98|-775.72| +2452638|50412|246|44047|586372|697|22622|1|229|3|96|98.02|136.24|104.90|0.00|10070.40|9409.92|13079.04|100.70|0.00|10070.40|10171.10|660.48| +||1531||||22622|1|222|3|||130.66|||3104.46||3527.82|124.17||||| +2452638|50412|3525|44047|586372|697|22622|1|165|3|42|97.03|178.53|7.14|0.00|299.88|4075.26|7498.26|17.99|0.00|299.88|317.87|-3775.38| +2452638|50412|13021|44047|586372|697|22622|1|56|3|64|74.69|120.25|2.40|0.00|153.60|4780.16|7696.00|3.07|0.00|153.60|156.67|-4626.56| +2452638|50412|17263|44047|586372|697|22622|1|14|3|17|72.38|111.46|75.79|0.00|1288.43|1230.46|1894.82|38.65|0.00|1288.43|1327.08|57.97| +2452638|50412|17971|44047|586372|697|22622|1|51|3|88|27.95|38.57|26.99|403.77|2375.12|2459.60|3394.16|0.00|403.77|1971.35|1971.35|-488.25| +2452638|50412|14443|44047|586372|697|22622|1|296|3|4|95.75|146.49|43.94|0.00|175.76|383.00|585.96|5.27|0.00|175.76|181.03|-207.24| +2452638|50412|14100|44047|586372|697|22622|1|77|3|79|44.66|83.06|17.44|1115.98|1377.76|3528.14|6561.74|5.23|1115.98|261.78|267.01|-3266.36| +2452638|50412|15786|44047|586372|697|22622|1|126|3|56|4.31|6.76|0.06|3.22|3.36|241.36|378.56|0.01|3.22|0.14|0.15|-241.22| +2452638|50412|10693|44047|586372|697|22622|1|140|3|27|37.04|58.89|55.35|0.00|1494.45|1000.08|1590.03|29.88|0.00|1494.45|1524.33|494.37| +2452638|50412|16869|44047|586372|697|22622|1|37|3|4|75.67|130.15|35.14|0.00|140.56|302.68|520.60|9.83|0.00|140.56|150.39|-162.12| +2452638|50412|5355|44047|586372|697|22622|1|90|3|53|23.04|44.00|26.40|0.00|1399.20|1221.12|2332.00|13.99|0.00|1399.20|1413.19|178.08| +2452638|50412|4698|44047|586372|697|22622|1|130|3|98|85.00|132.60|90.16|4506.19|8835.68|8330.00|12994.80|0.00|4506.19|4329.49|4329.49|-4000.51| +2451438|38195|12790|38814|874786|1511|19083|2|77|4|43|54.34|107.59|87.14|0.00|3747.02|2336.62|4626.37|299.76|0.00|3747.02|4046.78|1410.40| +2451438|38195|5515|38814|874786|1511|19083|2|224|4|23|2.91|3.57|3.53|32.47|81.19|66.93|82.11|0.97|32.47|48.72|49.69|-18.21| +2451438|38195|7990|38814|874786|1511|19083|2|16|4|56|64.68|78.90|24.45|0.00|1369.20|3622.08|4418.40|0.00|0.00|1369.20|1369.20|-2252.88| +2451438|38195|9685|38814|874786|1511|19083|2|284|4|21|40.39|40.39|12.11|0.00|254.31|848.19|848.19|22.88|0.00|254.31|277.19|-593.88| +2451438|38195|11036|38814|874786|1511|19083|2|229|4|41|67.18|78.60|5.50|0.00|225.50|2754.38|3222.60|13.53|0.00|225.50|239.03|-2528.88| +2451438|38195|8452|38814|874786|1511|19083|2|152|4|27|26.21|44.81|19.26|0.00|520.02|707.67|1209.87|46.80|0.00|520.02|566.82|-187.65| +2451438|38195|5350|38814|874786|1511|19083|2|241|4|86|64.67|123.51|55.57|0.00|4779.02|5561.62|10621.86|47.79|0.00|4779.02|4826.81|-782.60| +2451438|38195|163|38814|874786|1511|19083|2|190|4|17|54.26|105.80|9.52|116.52|161.84|922.42|1798.60|0.00|116.52|45.32|45.32|-877.10| +2451438|38195|1576|38814|874786|1511|19083|2|150|4|74|81.81|163.62|45.81|0.00|3389.94|6053.94|12107.88|271.19|0.00|3389.94|3661.13|-2664.00| +2451438|38195|6988|38814|874786|1511|19083|2|122|4|23|53.28|77.78|1.55|21.03|35.65|1225.44|1788.94|1.16|21.03|14.62|15.78|-1210.82| +2451465|53976|1940|13246|963544|5042|28940|2|70|5|60|69.54|112.65|57.45|0.00|3447.00|4172.40|6759.00|34.47|0.00|3447.00|3481.47|-725.40| +2451465|53976|11324|13246|963544|5042|28940|2|196|5|52|16.33|32.33|31.68|0.00|1647.36|849.16|1681.16|115.31|0.00|1647.36|1762.67|798.20| +2451465|53976|14599|13246|963544|5042|28940|2|167|5|83|74.15|89.72|63.70|0.00|5287.10|6154.45|7446.76|422.96|0.00|5287.10|5710.06|-867.35| +2451465|53976|11590|13246|963544|5042|28940|2|84|5|15|21.21|33.93|30.53|0.00|457.95|318.15|508.95|22.89|0.00|457.95|480.84|139.80| +2451465|53976|16792|13246|963544|5042|28940|2|12|5|71|22.10|37.57|28.17|0.00|2000.07|1569.10|2667.47|180.00|0.00|2000.07|2180.07|430.97| +2451465|53976|6466|13246|963544|5042|28940|2|200|5|36|7.93|12.92|7.36|0.00|264.96|285.48|465.12|0.00|0.00|264.96|264.96|-20.52| +2451465|53976|15752|13246|963544|5042|28940|2|10|5|66|71.06|136.43|45.02|0.00|2971.32|4689.96|9004.38|59.42|0.00|2971.32|3030.74|-1718.64| +2451465|53976|12650|13246|963544|5042|28940|2|185|5|66|21.01|28.57|5.99|0.00|395.34|1386.66|1885.62|7.90|0.00|395.34|403.24|-991.32| +|53976|5842||963544|||2|158|5|50|30.69|59.84||||1534.50||0.00||1735.00||200.50| +2451465|53976|14686|13246|963544|5042|28940|2|215|5|91|27.68|53.69|0.53|0.00|48.23|2518.88|4885.79|3.85|0.00|48.23|52.08|-2470.65| +2451465|53976|6068|13246|963544|5042|28940|2|125|5|76|89.78|103.24|50.58|0.00|3844.08|6823.28|7846.24|153.76|0.00|3844.08|3997.84|-2979.20| +2451465|53976|13562|13246|963544|5042|28940|2|75|5|64|87.90|95.81|66.10|0.00|4230.40|5625.60|6131.84|0.00|0.00|4230.40|4230.40|-1395.20| +2451465|53976|13958|13246|963544|5042|28940|2|149|5|60|41.72|52.98|32.31|0.00|1938.60|2503.20|3178.80|0.00|0.00|1938.60|1938.60|-564.60| +|53976|1808|13246|||28940|2||5|||77.52|11.62|0.00|732.06||4883.76|43.92|0.00|||| +2451465|53976|16195|13246|963544|5042|28940|2|206|5|50|30.96|56.34|45.07|0.00|2253.50|1548.00|2817.00|22.53|0.00|2253.50|2276.03|705.50| +||8621|53029||2092|||168|6||||||449.88|||||||| +2452109|36794|3049|53029|506184|2092|44800|4|72|6|31|49.78|77.15|69.43|0.00|2152.33|1543.18|2391.65|0.00|0.00|2152.33|2152.33|609.15| +2452109|36794|2549|53029|506184|2092|44800|4|128|6|62|85.07|159.08|76.35|0.00|4733.70|5274.34|9862.96|189.34|0.00|4733.70|4923.04|-540.64| +2452109|36794|10895|53029|506184|2092|44800|4|276|6|1|71.10|89.58|61.81|0.00|61.81|71.10|89.58|3.70|0.00|61.81|65.51|-9.29| +2452109|36794|11705|53029|506184|2092|44800|4|9|6|61|48.18|60.70|18.81|126.21|1147.41|2938.98|3702.70|71.48|126.21|1021.20|1092.68|-1917.78| +2452109|36794|13513|53029|506184|2092|44800|4|248|6|42|64.46|70.26|32.31|0.00|1357.02|2707.32|2950.92|27.14|0.00|1357.02|1384.16|-1350.30| +2452109|36794|3291|53029|506184|2092|44800|4|153|6|100|41.08|42.31|38.50|0.00|3850.00|4108.00|4231.00|346.50|0.00|3850.00|4196.50|-258.00| +2452109|36794|13245|53029|506184|2092|44800|4|252|6|64|86.35|152.83|122.26|6650.94|7824.64|5526.40|9781.12|35.21|6650.94|1173.70|1208.91|-4352.70| +2452109|36794|10355|53029|506184|2092|44800|4|216|6|94|62.67|99.64|97.64|0.00|9178.16|5890.98|9366.16|91.78|0.00|9178.16|9269.94|3287.18| +2452109|36794|6437|53029|506184|2092|44800|4|198|6|72|55.49|93.77|55.32|0.00|3983.04|3995.28|6751.44|199.15|0.00|3983.04|4182.19|-12.24| +2452109|36794|2647|53029|506184|2092|44800|4|150|6|100|4.45|7.87|7.24|0.00|724.00|445.00|787.00|57.92|0.00|724.00|781.92|279.00| +2452260|46712|5183|69954|890396|791|37633|2|6|7|51|84.65|124.43|49.77|0.00|2538.27|4317.15|6345.93|76.14|0.00|2538.27|2614.41|-1778.88| +2452260|46712|4627|69954|890396|791|37633|2|210|7|9|56.13|59.49|30.93|0.00|278.37|505.17|535.41|22.26|0.00|278.37|300.63|-226.80| +2452260||9751|69954|890396|791||||7|4|||||278.60|203.68|366.60|22.28||278.60|300.88|| +2452260|46712|15333|69954|||||85|7|||72.02||||||||4990.30||1141.70| +2452260|46712|4833|69954|890396|791|37633|2|153|7|88|38.23|72.63|39.22|0.00|3451.36|3364.24|6391.44|138.05|0.00|3451.36|3589.41|87.12| +2452260|46712|10487|69954|890396|791|37633|2|153|7|52|63.80|81.02|22.68|0.00|1179.36|3317.60|4213.04|23.58|0.00|1179.36|1202.94|-2138.24| +2452260|46712|5905|69954|890396|791|37633|2|77|7|69|99.85|113.82|110.40|0.00|7617.60|6889.65|7853.58|0.00|0.00|7617.60|7617.60|727.95| +2452260|46712|8955|69954|890396|791|37633|2|55|7|54|42.82|56.09|21.31|0.00|1150.74|2312.28|3028.86|92.05|0.00|1150.74|1242.79|-1161.54| +2452260|46712|15179|69954|890396|791|37633|2|5|7|12|47.60|75.20|65.42|0.00|785.04|571.20|902.40|47.10|0.00|785.04|832.14|213.84| +2452260|46712|4795|69954|890396|791|37633|2|212|7|73|12.17|14.23|11.24|623.59|820.52|888.41|1038.79|5.90|623.59|196.93|202.83|-691.48| +2452260|46712|17255|69954|890396|791|37633|2|275|7|69|34.19|58.46|50.86|0.00|3509.34|2359.11|4033.74|210.56|0.00|3509.34|3719.90|1150.23| +2452260|46712|12571|69954|890396|791|37633|2|147|7|82|69.53|87.60|34.16|0.00|2801.12|5701.46|7183.20|0.00|0.00|2801.12|2801.12|-2900.34| +2452179|52209|4183|86944|578318|6045|49019|8|109|8|90|81.63|151.83|113.87|9428.43|10248.30|7346.70|13664.70|40.99|9428.43|819.87|860.86|-6526.83| +2452179|52209|13427|86944|578318|6045|49019|8|106|8|87|31.78|63.24|16.44|0.00|1430.28|2764.86|5501.88|57.21|0.00|1430.28|1487.49|-1334.58| +2452179|52209|665|86944|578318|6045|49019|8|123|8|31|15.64|26.43|13.21|0.00|409.51|484.84|819.33|4.09|0.00|409.51|413.60|-75.33| +2452179|52209|11893|86944|578318|6045|49019|8|71|8|95|21.29|33.85|20.64|0.00|1960.80|2022.55|3215.75|98.04|0.00|1960.80|2058.84|-61.75| +2452179|52209|11365|86944|578318|6045|49019|8|239|8|7|18.65|27.97|20.41|0.00|142.87|130.55|195.79|1.42|0.00|142.87|144.29|12.32| +2452179|52209|7115|86944|578318|6045|49019|8|242|8|54|36.99|47.71|27.67|0.00|1494.18|1997.46|2576.34|89.65|0.00|1494.18|1583.83|-503.28| +2452179|52209|12041|86944|578318|6045|49019|8|299|8|95|91.80|93.63|75.84|6700.46|7204.80|8721.00|8894.85|5.04|6700.46|504.34|509.38|-8216.66| +2452179|52209|5929|86944|578318|6045|49019|8|128|8|83|14.11|27.37|3.28|0.00|272.24|1171.13|2271.71|13.61|0.00|272.24|285.85|-898.89| +2452179|52209|16671|86944|578318|6045|49019|8|45|8|20|18.95|33.92|31.88|0.00|637.60|379.00|678.40|51.00|0.00|637.60|688.60|258.60| +2452179|52209|17119|86944|578318|6045|49019|8|251|8|51|8.04|8.92|1.07|0.00|54.57|410.04|454.92|4.91|0.00|54.57|59.48|-355.47| +2452179|52209|17545|86944|578318|6045|49019|8|65|8|49|72.15|104.61|80.54|0.00|3946.46|3535.35|5125.89|39.46|0.00|3946.46|3985.92|411.11| +2452496|45986|12969|47480|1023064|4248|43385|4|14|9|59|88.31|106.85|44.87|0.00|2647.33|5210.29|6304.15|211.78|0.00|2647.33|2859.11|-2562.96| +2452496|45986|4623|47480|1023064|4248|43385|4|296|9|34|2.97|3.35|2.04|0.00|69.36|100.98|113.90|3.46|0.00|69.36|72.82|-31.62| +2452496|45986|17310|47480|1023064|4248|43385|4|167|9|33|27.13|53.44|2.13|0.00|70.29|895.29|1763.52|2.81|0.00|70.29|73.10|-825.00| +2452496|45986|13717|47480|1023064|4248|43385|4|172|9|53|75.37|89.69|54.71|0.00|2899.63|3994.61|4753.57|115.98|0.00|2899.63|3015.61|-1094.98| +|45986|16363|47480|1023064||43385|||9|54||||0.00|768.42|3873.42|||0.00||829.89|| +2452496|45986|69|47480|1023064|4248|43385|4|96|9|11|31.70|54.52|47.97|527.67|527.67|348.70|599.72|0.00|527.67|0.00|0.00|-348.70| +2452496|45986|16461|47480|1023064|4248|43385|4|281|9|66|15.21|26.00|25.22|66.58|1664.52|1003.86|1716.00|111.85|66.58|1597.94|1709.79|594.08| +2452496|45986|889|47480|1023064|4248|43385|4|287|9|6|27.17|51.62|25.29|0.00|151.74|163.02|309.72|3.03|0.00|151.74|154.77|-11.28| +2452496|45986|12027|47480|1023064|4248|43385|4|1|9|27|98.68|195.38|15.63|0.00|422.01|2664.36|5275.26|33.76|0.00|422.01|455.77|-2242.35| +2452496|45986|1185|47480|1023064|4248|43385|4|139|9|62|55.68|64.03|3.84|233.31|238.08|3452.16|3969.86|0.14|233.31|4.77|4.91|-3447.39| +2452496|45986|16083|47480|1023064|4248|43385|4|124|9|32|99.10|152.61|132.77|0.00|4248.64|3171.20|4883.52|339.89|0.00|4248.64|4588.53|1077.44| +2452496|45986|7945|47480|1023064|4248|43385|4|239|9|83|8.10|14.58|8.60|0.00|713.80|672.30|1210.14|64.24|0.00|713.80|778.04|41.50| +|45986|13483|||4248||4|109|9||59.14|108.81|46.78|0.00||3311.84|6093.36||0.00|2619.68|2619.68|| +2452496|45986|8334|47480|1023064|4248|43385|4|22|9|71|34.79|36.87|19.17|1197.74|1361.07|2470.09|2617.77|8.16|1197.74|163.33|171.49|-2306.76| +2452496|45986|15133|47480|1023064|4248|43385|4|111|9|15|35.89|63.88|30.66|0.00|459.90|538.35|958.20|41.39|0.00|459.90|501.29|-78.45| +2452496|45986|16659|47480|1023064|4248|43385|4|1|9|84|76.71|85.14|5.10|0.00|428.40|6443.64|7151.76|21.42|0.00|428.40|449.82|-6015.24| +2451966|60226|9811|4451|71288|6925|24515|10|38|10|28|47.85|50.24|19.09|0.00|534.52|1339.80|1406.72|37.41|0.00|534.52|571.93|-805.28| +2451966|60226|3933|4451|71288|6925|24515|10|61|10|52|52.19|66.80|26.05|0.00|1354.60|2713.88|3473.60|94.82|0.00|1354.60|1449.42|-1359.28| +2451966|60226|5863|4451|71288|6925|24515|10|27|10|47|11.71|22.60|3.61|0.00|169.67|550.37|1062.20|3.39|0.00|169.67|173.06|-380.70| +2451966|60226|4357|4451|71288|6925|24515|10|122|10|17|88.36|133.42|53.36|0.00|907.12|1502.12|2268.14|0.00|0.00|907.12|907.12|-595.00| +2451966|60226|15447|4451|71288|6925|24515|10|250|10|67|33.28|34.61|19.72|0.00|1321.24|2229.76|2318.87|39.63|0.00|1321.24|1360.87|-908.52| +2451966|60226|3433|4451|71288|6925|24515|10|195|10|83|10.26|17.33|9.18|0.00|761.94|851.58|1438.39|45.71|0.00|761.94|807.65|-89.64| +2451966|60226|755|4451|71288|6925|24515|10|91|10|74|82.24|152.96|97.89|0.00|7243.86|6085.76|11319.04|144.87|0.00|7243.86|7388.73|1158.10| +||1425|4451|71288|6925|24515|||10|92|||||2092.08|2280.68|3033.24|||2092.08|2280.36|-188.60| +2451966|60226|1511|4451|71288|6925|24515|10|95|10|76|31.47|56.01|31.36|0.00|2383.36|2391.72|4256.76|143.00|0.00|2383.36|2526.36|-8.36| +2451966|60226|13803|4451|71288|6925|24515|10|185|10|66|82.35|137.52|82.51|0.00|5445.66|5435.10|9076.32|0.00|0.00|5445.66|5445.66|10.56| +2452420|68961|7519|48900|781292|2484|22923|8|68|11|68|7.16|12.88|5.66|107.76|384.88|486.88|875.84|8.31|107.76|277.12|285.43|-209.76| +2452420|68961|1315|48900|781292|2484|22923|8|112|11|70|45.84|49.50|45.54|0.00|3187.80|3208.80|3465.00|159.39|0.00|3187.80|3347.19|-21.00| +2452420|68961|12438|48900|781292|2484|22923|8|239|11|16|92.94|185.88|46.47|0.00|743.52|1487.04|2974.08|44.61|0.00|743.52|788.13|-743.52| +2452420|68961|17395|48900|781292|2484|22923|8|214|11|85|38.04|42.98|22.34|0.00|1898.90|3233.40|3653.30|56.96|0.00|1898.90|1955.86|-1334.50| +2452420|68961|157|48900|781292|2484|22923|8|146|11|84|64.63|101.46|14.20|0.00|1192.80|5428.92|8522.64|59.64|0.00|1192.80|1252.44|-4236.12| +2452420|68961|10699|48900|781292|2484|22923|8|199|11|33|73.77|138.68|128.97|0.00|4256.01|2434.41|4576.44|0.00|0.00|4256.01|4256.01|1821.60| +2452420|68961|15157|48900|781292|2484|22923|8|90|11|96|15.52|21.41|21.41|0.00|2055.36|1489.92|2055.36|61.66|0.00|2055.36|2117.02|565.44| +2452420|68961|17226|48900|781292|2484|22923|8|68|11|11|34.03|44.23|42.46|0.00|467.06|374.33|486.53|18.68|0.00|467.06|485.74|92.73| +2452420|68961|11991|48900|781292|2484|22923|8|113|11|38|3.27|3.56|0.32|0.00|12.16|124.26|135.28|0.48|0.00|12.16|12.64|-112.10| +2452420|68961|9901|48900|781292|2484|22923|8|294|11|57|46.93|80.71|63.76|0.00|3634.32|2675.01|4600.47|327.08|0.00|3634.32|3961.40|959.31| +2452420|68961|7608|48900|781292|2484|22923|8|161|11|66|8.34|15.76|15.76|0.00|1040.16|550.44|1040.16|83.21|0.00|1040.16|1123.37|489.72| +2452420|68961|15649|48900|781292|2484|22923|8|277|11|33|66.11|114.37|78.91|0.00|2604.03|2181.63|3774.21|104.16|0.00|2604.03|2708.19|422.40| +||11490||781292||22923|8|246|11|||||0.00||4141.28||483.99|0.00||5861.67|1236.40| +2451206|35137|6248|20820|63209|3198|42434|8|261|12|36|75.17|75.92|40.23|0.00|1448.28|2706.12|2733.12|14.48|0.00|1448.28|1462.76|-1257.84| +2451206|35137|14944|20820|63209|3198|42434|8|1|12|9|7.33|11.94|11.22|31.30|100.98|65.97|107.46|2.09|31.30|69.68|71.77|3.71| +2451206|35137|9788|20820|63209|3198|42434|8|297|12|73|79.42|95.30|78.14|0.00|5704.22|5797.66|6956.90|456.33|0.00|5704.22|6160.55|-93.44| +2451206|35137|373|20820|63209|3198|42434|8|286|12|57|13.95|27.06|8.38|0.00|477.66|795.15|1542.42|33.43|0.00|477.66|511.09|-317.49| +2451206|35137|9616|20820|63209|3198|42434|8|79|12|66|99.06|104.01|78.00|0.00|5148.00|6537.96|6864.66|154.44|0.00|5148.00|5302.44|-1389.96| +2451206|35137|4888|20820|63209|3198|42434|8|183|12|56|75.74|133.30|0.00|0.00|0.00|4241.44|7464.80|0.00|0.00|0.00|0.00|-4241.44| +2451206|35137|14746|20820|63209|3198|42434|8|87|12|26|58.74|82.23|60.02|0.00|1560.52|1527.24|2137.98|46.81|0.00|1560.52|1607.33|33.28| +2451206|35137|1591|20820|63209|3198|42434|8|277|12|82|45.84|49.96|22.98|1262.52|1884.36|3758.88|4096.72|55.96|1262.52|621.84|677.80|-3137.04| +2451206|35137|6148|20820|63209|3198|42434|8|139|12|36|97.62|193.28|15.46|0.00|556.56|3514.32|6958.08|5.56|0.00|556.56|562.12|-2957.76| +2451206|35137|15440|20820|63209|3198|42434|8|1|12|99|27.09|53.63|26.27|0.00|2600.73|2681.91|5309.37|104.02|0.00|2600.73|2704.75|-81.18| +2451206|35137|13399|20820|63209|3198|42434|8|289|12|46|45.27|60.20|22.27|0.00|1024.42|2082.42|2769.20|51.22|0.00|1024.42|1075.64|-1058.00| +2450850|65476|5971|15144|1401973|4002|32431|1|138|13|80|72.61|82.04|63.17|0.00|5053.60|5808.80|6563.20|454.82|0.00|5053.60|5508.42|-755.20| +2450850|65476|6682|15144|1401973|4002|32431|1|135|13|80|32.05|41.98|22.66|0.00|1812.80|2564.00|3358.40|72.51|0.00|1812.80|1885.31|-751.20| +2450850|65476|4430|15144|1401973|4002|32431|1|120|13|73|5.86|11.72|0.70|0.00|51.10|427.78|855.56|4.08|0.00|51.10|55.18|-376.68| +2450850|65476|6085|15144|1401973|4002|32431|1|72|13|58|21.45|28.74|22.12|0.00|1282.96|1244.10|1666.92|115.46|0.00|1282.96|1398.42|38.86| +2450850|65476|1760|15144|1401973|4002|32431|1|121|13|12|80.96|138.44|41.53|0.00|498.36|971.52|1661.28|24.91|0.00|498.36|523.27|-473.16| +2450850|65476|10096|15144|1401973|4002|32431|1|107|13|12|17.14|30.85|28.99|0.00|347.88|205.68|370.20|6.95|0.00|347.88|354.83|142.20| +|65476|1898|15144|||32431|1|180|13||96.46|||0.00|432.48|385.84||8.64|0.00||441.12|| +2450850|65476|7640|15144|1401973|4002|32431|1|5|13|48|17.06|28.49|10.82|0.00|519.36|818.88|1367.52|25.96|0.00|519.36|545.32|-299.52| +2450850||868|15144||4002|32431|1||13||62.85|109.35||0.00||5467.95|9513.45|38.03|0.00|950.91||-4517.04| +2450850|65476|17240|15144|1401973|4002|32431|1|163|13|20|93.85|185.82|137.50|2255.00|2750.00|1877.00|3716.40|39.60|2255.00|495.00|534.60|-1382.00| +2450850|65476|16894|15144|1401973|4002|32431|1|255|13|87|20.99|30.43|16.12|1192.07|1402.44|1826.13|2647.41|14.72|1192.07|210.37|225.09|-1615.76| +2450850|65476|6140|15144|1401973|4002|32431|1|1|13|15|89.90|163.61|4.90|46.30|73.50|1348.50|2454.15|2.44|46.30|27.20|29.64|-1321.30| +2450850|65476|2191|15144||4002|32431|||13||||15.60|||2390.50||||||| +2450850|65476|7723|15144|1401973|4002|32431|1|252|13|27|59.09|112.86|83.51|0.00|2254.77|1595.43|3047.22|202.92|0.00|2254.77|2457.69|659.34| +2450850|65476|2108||1401973||32431|||13|9||||478.29||||1.89|478.29||49.20|| +2450850|65476|11758|15144|1401973|4002|32431|1|84|13|34|72.24|107.63|49.50|1464.21|1683.00|2456.16|3659.42|17.50|1464.21|218.79|236.29|-2237.37| +2452217|34167|13515|19409|1145357|4905|39653|4|125|14|57|87.32|92.55|43.49|1933.56|2478.93|4977.24|5275.35|5.45|1933.56|545.37|550.82|-4431.87| +2452217|34167|177|19409|1145357|4905|39653|4|15|14|41|13.05|15.13|9.22|109.62|378.02|535.05|620.33|0.00|109.62|268.40|268.40|-266.65| +2452217|34167|16929|19409|1145357|4905|39653|4|120|14|14|54.76|91.99|0.91|0.00|12.74|766.64|1287.86|0.25|0.00|12.74|12.99|-753.90| +2452217|34167|16741|19409|1145357|4905|39653|4|253|14|46|76.43|110.82|64.27|0.00|2956.42|3515.78|5097.72|236.51|0.00|2956.42|3192.93|-559.36| +2452217|34167|11549|19409|1145357|4905|39653|4|254|14|6|19.33|26.67|20.80|0.00|124.80|115.98|160.02|7.48|0.00|124.80|132.28|8.82| +2452217|34167|11817|19409|1145357|4905|39653|4|182|14|81|60.77|66.84|7.35|0.00|595.35|4922.37|5414.04|35.72|0.00|595.35|631.07|-4327.02| +2452217|34167|11653|19409|1145357|4905|39653|4|18|14|60|86.94|88.67|39.01|1638.42|2340.60|5216.40|5320.20|0.00|1638.42|702.18|702.18|-4514.22| +2452217||12587|19409||4905|||82|14||||||1233.75||3163.50|74.02||||-467.25| +2452217|34167|13069|19409|1145357|4905|39653|4|117|14|77|93.60|105.76|68.74|0.00|5292.98|7207.20|8143.52|264.64|0.00|5292.98|5557.62|-1914.22| +2452217|34167|4507|19409|1145357|4905|39653|4|118|14|4|45.45|84.08|65.58|0.00|262.32|181.80|336.32|0.00|0.00|262.32|262.32|80.52| +2452217|34167|13845|19409|1145357|4905|39653|4|146|14|17|52.30|84.72|64.38|0.00|1094.46|889.10|1440.24|76.61|0.00|1094.46|1171.07|205.36| +2452217|34167|10175|19409|1145357|4905|39653|4|234|14|19|39.97|57.15|46.86|0.00|890.34|759.43|1085.85|53.42|0.00|890.34|943.76|130.91| +2452217|34167|769|19409|1145357|4905|39653|4|258|14|20|26.29|51.79|9.32|0.00|186.40|525.80|1035.80|11.18|0.00|186.40|197.58|-339.40| +2451797|55556|17267|62835|227170|1458|32993|4|70|15|7|30.61|51.73|51.73|0.00|362.11|214.27|362.11|25.34|0.00|362.11|387.45|147.84| +2451797|55556|11659|62835|227170|1458|32993|4|197|15|80|86.23|156.93|20.40|750.72|1632.00|6898.40|12554.40|17.62|750.72|881.28|898.90|-6017.12| +2451797|55556|17857|62835|227170|1458|32993|4|291|15|38|96.35|104.05|32.25|0.00|1225.50|3661.30|3953.90|61.27|0.00|1225.50|1286.77|-2435.80| +2451797|55556|4777|62835|227170|1458|32993|4|4|15|28|36.86|58.60|58.01|0.00|1624.28|1032.08|1640.80|32.48|0.00|1624.28|1656.76|592.20| +2451797|55556|8690|62835|227170|1458|32993|4|148|15|32|67.37|117.89|9.43|0.00|301.76|2155.84|3772.48|24.14|0.00|301.76|325.90|-1854.08| +2451797|55556|10361|62835|227170|1458|32993|4|82|15|39|74.88|125.04|1.25|0.00|48.75|2920.32|4876.56|2.92|0.00|48.75|51.67|-2871.57| +2451797|55556|16619|62835|227170|1458|32993|4|133|15|81|80.21|152.39|105.14|0.00|8516.34|6497.01|12343.59|766.47|0.00|8516.34|9282.81|2019.33| +2451797|55556|8707|62835|227170|1458|32993|4|58|15|21|48.54|57.27|1.14|0.00|23.94|1019.34|1202.67|0.23|0.00|23.94|24.17|-995.40| +2451797|55556|8353|62835|||32993|4|157|15|||||0.00||||28.50|0.00|950.04||-1336.79| +2451797|55556|4505|62835|227170|1458|32993|4|142|15|59|77.35|113.70|61.39|0.00|3622.01|4563.65|6708.30|289.76|0.00|3622.01|3911.77|-941.64| +2451797|55556|4241|62835|227170|1458|32993|4|136|15|21|89.07|132.71|95.55|0.00|2006.55|1870.47|2786.91|120.39|0.00|2006.55|2126.94|136.08| +2451797|55556|17330|62835|227170|1458|32993|4|265|15|82|67.45|122.08|95.22|6714.91|7808.04|5530.90|10010.56|76.51|6714.91|1093.13|1169.64|-4437.77| +2451797|55556|7391|62835|227170|1458|32993|4|32|15|98|53.76|65.58|3.27|0.00|320.46|5268.48|6426.84|0.00|0.00|320.46|320.46|-4948.02| +2451797|55556|8336|62835|227170|1458|32993|4|112|15|15|44.09|60.84|45.02|0.00|675.30|661.35|912.60|13.50|0.00|675.30|688.80|13.95| +2451797|55556|17564|62835|227170|1458|32993|4|136|15|26|63.52|88.92|40.90|0.00|1063.40|1651.52|2311.92|0.00|0.00|1063.40|1063.40|-588.12| +2451797|55556|13172|62835|227170|1458|32993|4|35|15|25|47.11|57.94|38.81|0.00|970.25|1177.75|1448.50|29.10|0.00|970.25|999.35|-207.50| +2450872|47818|9997|51139|263380|868|33509|1|270|16|94|64.76|119.80|74.27|0.00|6981.38|6087.44|11261.20|488.69|0.00|6981.38|7470.07|893.94| +2450872|47818|17246|51139|263380|868|33509|1|51|16|70|80.85|161.70|100.25|0.00|7017.50|5659.50|11319.00|210.52|0.00|7017.50|7228.02|1358.00| +2450872|47818|9790|51139|263380|868|33509|1|4|16|17|36.95|61.70|60.46|0.00|1027.82|628.15|1048.90|20.55|0.00|1027.82|1048.37|399.67| +2450872|47818|4144|51139|263380|868|33509|1|228|16|94|19.91|25.28|22.49|0.00|2114.06|1871.54|2376.32|21.14|0.00|2114.06|2135.20|242.52| +2450872|47818|8887|51139||868||||16|35|59.82||50.91|||2093.70|2784.60||||1870.94|-311.85| +2450872|47818|6008|51139|263380|868|33509|1|191|16|59|59.62|63.79|60.60|0.00|3575.40|3517.58|3763.61|214.52|0.00|3575.40|3789.92|57.82| +2450872|47818|457|51139|263380|868|33509|1|248|16|60|91.53|167.49|68.67|0.00|4120.20|5491.80|10049.40|370.81|0.00|4120.20|4491.01|-1371.60| +2450872|47818|11168|51139|263380|868|33509|1|132|16|86|62.85|103.07|64.93|0.00|5583.98|5405.10|8864.02|390.87|0.00|5583.98|5974.85|178.88| +2450872|47818|9769|51139|263380|868|33509|1|267|16|42|29.53|33.66|12.11|391.63|508.62|1240.26|1413.72|7.01|391.63|116.99|124.00|-1123.27| +2450872|47818|7504|51139|263380|868|33509|1|172|16|51|31.35|55.17|15.99|0.00|815.49|1598.85|2813.67|65.23|0.00|815.49|880.72|-783.36| +2450872|47818|1888|51139|263380|868|33509|1|48|16|4|47.64|61.93|55.73|0.00|222.92|190.56|247.72|6.68|0.00|222.92|229.60|32.36| +2450872|47818|16226|51139|263380|868|33509|1|152|16|13|31.30|61.03|31.12|0.00|404.56|406.90|793.39|36.41|0.00|404.56|440.97|-2.34| +2450872|47818|11920|51139|263380|868|33509|1|145|16|29|94.31|151.83|103.24|0.00|2993.96|2734.99|4403.07|29.93|0.00|2993.96|3023.89|258.97| +2451200|49168|17683|77683|756435|2034|22386|7|185|17|34|81.48|136.07|55.78|0.00|1896.52|2770.32|4626.38|37.93|0.00|1896.52|1934.45|-873.80| +2451200|49168|4678|77683|756435|2034|22386|7|68|17|34|32.47|56.49|44.06|149.80|1498.04|1103.98|1920.66|121.34|149.80|1348.24|1469.58|244.26| +2451200|49168|6811|77683|756435|2034|22386|7|144|17|70|62.96|66.73|24.02|0.00|1681.40|4407.20|4671.10|33.62|0.00|1681.40|1715.02|-2725.80| +2451200|49168|13177|77683|756435|2034|22386|7|209|17|45|44.18|72.45|51.43|0.00|2314.35|1988.10|3260.25|138.86|0.00|2314.35|2453.21|326.25| +2451200|49168|2092|77683|756435|2034|22386|7|110|17|37|31.71|47.88|37.82|1385.34|1399.34|1173.27|1771.56|0.00|1385.34|14.00|14.00|-1159.27| +2451200|49168|11203|77683|756435|2034|22386|7|80|17|21|93.44|172.86|31.11|0.00|653.31|1962.24|3630.06|26.13|0.00|653.31|679.44|-1308.93| +2451200|49168|13826|77683|756435|2034|22386|7|188|17|32|58.61|63.88|47.91|0.00|1533.12|1875.52|2044.16|137.98|0.00|1533.12|1671.10|-342.40| +2451200|49168|15781|77683|756435|2034|22386|7|279|17|76|24.79|33.46|4.34|0.00|329.84|1884.04|2542.96|6.59|0.00|329.84|336.43|-1554.20| +2451200|49168|9214|77683|756435|2034|22386|7|212|17|57|14.20|25.56|8.94|0.00|509.58|809.40|1456.92|25.47|0.00|509.58|535.05|-299.82| +2451200|49168|10543|77683|756435|2034|22386|7|32|17|54|57.11|93.66|76.80|0.00|4147.20|3083.94|5057.64|290.30|0.00|4147.20|4437.50|1063.26| +2451415|54836|8884|63292|54321|997|9226|10|55|18|18|43.49|59.58|12.51|0.00|225.18|782.82|1072.44|18.01|0.00|225.18|243.19|-557.64| +2451415|54836|11584|63292|54321|997|9226|10|289|18|76|4.26|7.58|3.56|0.00|270.56|323.76|576.08|13.52|0.00|270.56|284.08|-53.20| +2451415|54836|11890|63292|54321|997|9226|10|99|18|7|82.36|159.77|63.90|0.00|447.30|576.52|1118.39|4.47|0.00|447.30|451.77|-129.22| +2451415|54836|12602|63292|54321|997|9226|10|162|18|81|11.32|18.79|8.07|0.00|653.67|916.92|1521.99|0.00|0.00|653.67|653.67|-263.25| +2451415|54836|7378|63292|54321|997|9226|10|270|18|94|61.01|114.08|99.24|7835.99|9328.56|5734.94|10723.52|119.40|7835.99|1492.57|1611.97|-4242.37| +2451415|54836|9886|63292|54321|997|9226|10|191|18|62|9.59|15.43|10.49|0.00|650.38|594.58|956.66|32.51|0.00|650.38|682.89|55.80| +2451415|54836|14011|63292|54321|997|9226|10|221|18|95|55.01|101.21|36.43|0.00|3460.85|5225.95|9614.95|311.47|0.00|3460.85|3772.32|-1765.10| +2451415|54836|17995|63292|54321|997|9226|10|30|18|13|46.79|52.87|32.25|327.01|419.25|608.27|687.31|2.76|327.01|92.24|95.00|-516.03| +2451415|54836|12860|63292|54321|997|9226|10|221|18|18|19.89|32.61|4.23|0.00|76.14|358.02|586.98|4.56|0.00|76.14|80.70|-281.88| +2451415|54836|12826|63292|54321|997|9226|10|89|18|93|82.82|135.82|35.31|426.89|3283.83|7702.26|12631.26|142.84|426.89|2856.94|2999.78|-4845.32| +2451415|54836|14377|63292|54321|997|9226|10|181|18|15|15.47|20.57|18.10|203.62|271.50|232.05|308.55|0.00|203.62|67.88|67.88|-164.17| +2451415|54836|2440|63292|54321|997|9226|10|142|18|40|15.39|28.93|24.87|0.00|994.80|615.60|1157.20|49.74|0.00|994.80|1044.54|379.20| +2451415|54836|8779|63292|54321|||10||18|9|75.19|121.80|60.90|0.00|||||0.00|||| +2451415|54836|14372|63292|54321|997|9226|10|211|18|76|89.58|130.78|27.46|0.00|2086.96|6808.08|9939.28|83.47|0.00|2086.96|2170.43|-4721.12| +2451415|54836|5251|63292|54321|997|9226|10|163|18|41|45.83|78.36|38.39|1353.63|1573.99|1879.03|3212.76|15.42|1353.63|220.36|235.78|-1658.67| +2451361|65446|11914|51104|1847200|1343|38239|7|287|19|59|88.03|153.17|13.78|0.00|813.02|5193.77|9037.03|56.91|0.00|813.02|869.93|-4380.75| +2451361|65446|10651|51104|1847200|1343|38239|7|196|19|46|57.69|70.95|7.09|104.36|326.14|2653.74|3263.70|19.96|104.36|221.78|241.74|-2431.96| +2451361|65446|3133|51104|1847200|1343|38239|7|56|19|96|68.89|102.64|25.66|0.00|2463.36|6613.44|9853.44|221.70|0.00|2463.36|2685.06|-4150.08| +2451361|65446|7087|51104|1847200|1343|38239|7|266|19|1|48.67|51.10|31.68|0.00|31.68|48.67|51.10|1.26|0.00|31.68|32.94|-16.99| +2451361|65446|9346|51104|1847200|1343|38239|7|202|19|39|84.06|85.74|0.85|0.00|33.15|3278.34|3343.86|2.98|0.00|33.15|36.13|-3245.19| +2451361|65446|6772|51104|1847200|1343|38239|7|99|19|97|36.04|62.34|21.81|0.00|2115.57|3495.88|6046.98|21.15|0.00|2115.57|2136.72|-1380.31| +||9094|51104|||38239||169|19|49|61.82|110.03|||1401.40||5391.47|||||| +2451361|65446|1094|51104|1847200|1343|38239|7|74|19|48|19.55|34.79|20.52|128.04|984.96|938.40|1669.92|51.41|128.04|856.92|908.33|-81.48| +2451361|65446|3376|51104|1847200|1343|38239|7|299|19|84|63.07|90.19|30.66|0.00|2575.44|5297.88|7575.96|51.50|0.00|2575.44|2626.94|-2722.44| +2451361|65446|4882|51104|1847200|1343|38239|7|182|19|84|41.48|44.79|33.14|0.00|2783.76|3484.32|3762.36|27.83|0.00|2783.76|2811.59|-700.56| +2451361|65446|8662|51104|1847200|1343|38239|7|163|19|97|72.78|90.24|83.02|0.00|8052.94|7059.66|8753.28|724.76|0.00|8052.94|8777.70|993.28| +2451361|65446|17539|51104|1847200|1343|38239|7|68|19|20|69.20|93.42|73.80|0.00|1476.00|1384.00|1868.40|14.76|0.00|1476.00|1490.76|92.00| +2451361|65446|16330|51104|||38239|7|278|19||79.15||47.10|0.00||395.75||9.42|0.00||244.92|| +2451361|65446|7814|51104|1847200|1343|38239|7|297|19|29|61.78|100.70|16.11|0.00|467.19|1791.62|2920.30|42.04|0.00|467.19|509.23|-1324.43| +2451361|65446|10558|51104|1847200|1343|38239|7|289|19|82|12.34|17.52|6.13|0.00|502.66|1011.88|1436.64|40.21|0.00|502.66|542.87|-509.22| +2451556|45944|5425|44828|671305|3382|48259|10|250|20|15|28.19|29.88|5.97|11.64|89.55|422.85|448.20|5.45|11.64|77.91|83.36|-344.94| +2451556|45944|6026|44828|671305|3382|48259|10|144|20|21|80.56|99.89|15.98|0.00|335.58|1691.76|2097.69|6.71|0.00|335.58|342.29|-1356.18| +2451556|45944|13601|44828|671305|3382|48259|10|133|20|17|1.40|2.52|0.80|11.83|13.60|23.80|42.84|0.12|11.83|1.77|1.89|-22.03| +2451556|45944|2618|44828|671305|3382|48259|10|236|20|4|69.47|121.57|71.72|106.14|286.88|277.88|486.28|16.26|106.14|180.74|197.00|-97.14| +2451556|45944|15131|44828|671305|3382|48259|10|188|20|85|42.56|68.94|19.99|0.00|1699.15|3617.60|5859.90|50.97|0.00|1699.15|1750.12|-1918.45| +2451556|45944|9800|44828|671305|3382|48259|10|230|20|32|18.09|32.74|31.10|0.00|995.20|578.88|1047.68|39.80|0.00|995.20|1035.00|416.32| +|45944|14935|44828||||10|241|20|||22.03|7.04||337.92|719.52||10.13||337.92|348.05|| +2451556|45944|9086|44828|671305|3382|48259|10|12|20|4|98.99|188.08|0.00|0.00|0.00|395.96|752.32|0.00|0.00|0.00|0.00|-395.96| +2451556|45944|5312|44828|671305|3382|48259|10|297|20|9|29.45|50.06|28.03|0.00|252.27|265.05|450.54|5.04|0.00|252.27|257.31|-12.78| +2451556|45944|1451|44828|671305|3382|48259|10|188|20|89|84.34|164.46|18.09|0.00|1610.01|7506.26|14636.94|80.50|0.00|1610.01|1690.51|-5896.25| +2451556|45944|5483|44828|671305|3382|48259|10|103|20|8|30.74|33.50|11.39|0.00|91.12|245.92|268.00|3.64|0.00|91.12|94.76|-154.80| +|45944|8714|44828|671305|3382|48259||97|20||8.15|||0.00|112.44|97.80|190.68|4.49|0.00|||14.64| +2451556|45944|7207|44828|671305|3382|48259|10|130|20|90|83.12|102.23|68.49|0.00|6164.10|7480.80|9200.70|308.20|0.00|6164.10|6472.30|-1316.70| +2451172|70722|2870|60478|1673919|2560|10401|1|84|21|50|91.94|138.82|27.76|0.00|1388.00|4597.00|6941.00|69.40|0.00|1388.00|1457.40|-3209.00| +2451172|70722|12266|60478|1673919|2560|10401|1|172|21|47|11.55|12.58|12.20|0.00|573.40|542.85|591.26|51.60|0.00|573.40|625.00|30.55| +2451172|70722|230|60478|1673919|2560|10401|1|280|21|48|13.37|23.93|20.10|0.00|964.80|641.76|1148.64|86.83|0.00|964.80|1051.63|323.04| +2451172|70722|9187|60478|1673919|2560|10401|1|115|21|14|35.49|42.58|24.69|214.30|345.66|496.86|596.12|3.94|214.30|131.36|135.30|-365.50| +2451172|70722|6476|60478|1673919|2560|10401|1|89|21|49|20.29|31.24|6.24|0.00|305.76|994.21|1530.76|3.05|0.00|305.76|308.81|-688.45| +2451172|70722|1810|60478|1673919|2560|10401|1|88|21|59|66.37|126.76|20.28|873.45|1196.52|3915.83|7478.84|0.00|873.45|323.07|323.07|-3592.76| +2451172|70722|5170|60478|1673919|2560|10401|1|93|21|45|90.00|108.00|65.88|0.00|2964.60|4050.00|4860.00|88.93|0.00|2964.60|3053.53|-1085.40| +2451172|70722|5998|60478|1673919|2560|10401|1|13|21|51|9.41|10.44|3.02|150.93|154.02|479.91|532.44|0.00|150.93|3.09|3.09|-476.82| +2451172|70722|14368|60478|1673919|2560|10401|1|275|21|18|51.29|69.24|59.54|664.46|1071.72|923.22|1246.32|36.65|664.46|407.26|443.91|-515.96| +2451172|70722|14396|60478|1673919|2560|10401|1|109|21|88|45.26|54.76|15.33|0.00|1349.04|3982.88|4818.88|13.49|0.00|1349.04|1362.53|-2633.84| +2451112|65253|16396|64935|1392098|4508|11671|10|146|22|85|86.46|129.69|25.93|0.00|2204.05|7349.10|11023.65|44.08|0.00|2204.05|2248.13|-5145.05| +2451112|65253|17660|64935|1392098|4508|11671|10|102|22|70|95.56|132.82|31.87|0.00|2230.90|6689.20|9297.40|66.92|0.00|2230.90|2297.82|-4458.30| +2451112|65253|15310|64935|1392098|4508|11671|10|154|22|15|82.24|137.34|108.49|1594.80|1627.35|1233.60|2060.10|2.60|1594.80|32.55|35.15|-1201.05| +2451112|65253|12415|64935|1392098|4508|11671|10|160|22|10|38.97|41.69|5.83|1.74|58.30|389.70|416.90|0.00|1.74|56.56|56.56|-333.14| +2451112|65253|9985|64935|1392098|4508|11671|10|145|22|70|21.46|22.96|2.06|0.00|144.20|1502.20|1607.20|0.00|0.00|144.20|144.20|-1358.00| +2451112|65253|17392|64935|1392098|4508|11671|10|262|22|14|51.86|91.27|73.01|0.00|1022.14|726.04|1277.78|30.66|0.00|1022.14|1052.80|296.10| +2451112|65253|16922|64935|1392098|4508|11671|10|190|22|88|28.00|29.68|18.99|0.00|1671.12|2464.00|2611.84|133.68|0.00|1671.12|1804.80|-792.88| +2451112|65253|11599|64935|1392098|4508|11671|10|161|22|66|5.01|5.01|3.10|0.00|204.60|330.66|330.66|6.13|0.00|204.60|210.73|-126.06| +2451112|65253|10474|64935|1392098|4508|11671|10|236|22|31|45.65|54.32|51.60|0.00|1599.60|1415.15|1683.92|0.00|0.00|1599.60|1599.60|184.45| +2452617|31683|7242|21669|1833048|6474|33053|10|189|23|37|54.82|55.91|0.55|18.31|20.35|2028.34|2068.67|0.06|18.31|2.04|2.10|-2026.30| +2452617|31683|11257|21669|1833048|6474|33053|10|142|23|64|29.95|45.52|8.64|0.00|552.96|1916.80|2913.28|0.00|0.00|552.96|552.96|-1363.84| +2452617|31683|12397|21669|1833048|6474|33053|10|106|23|80|78.73|103.13|20.62|0.00|1649.60|6298.40|8250.40|115.47|0.00|1649.60|1765.07|-4648.80| +2452617|31683|9093|21669|1833048|6474|33053|10|52|23|38|80.20|81.00|22.68|0.00|861.84|3047.60|3078.00|17.23|0.00|861.84|879.07|-2185.76| +2452617|31683|15403|21669|1833048|6474|33053|10|153|23|96|51.96|77.42|73.54|0.00|7059.84|4988.16|7432.32|352.99|0.00|7059.84|7412.83|2071.68| +2452617|31683|8181|21669|1833048|6474|33053|10|138|23|13|4.63|7.96|6.60|0.00|85.80|60.19|103.48|0.00|0.00|85.80|85.80|25.61| +2452617|31683|17631|21669|1833048|6474|33053|10|226|23|16|22.06|39.48|34.34|0.00|549.44|352.96|631.68|16.48|0.00|549.44|565.92|196.48| +2452617|31683|319|21669|1833048|6474|33053|10|80|23|86|66.36|104.18|59.38|0.00|5106.68|5706.96|8959.48|255.33|0.00|5106.68|5362.01|-600.28| +2452617|31683|11220|21669|1833048|6474|33053|10|277|23|91|71.30|89.12|36.53|0.00|3324.23|6488.30|8109.92|99.72|0.00|3324.23|3423.95|-3164.07| +2452617|31683|8413|21669|1833048|6474|33053|10|260|23|1|14.20|14.91|3.42|0.54|3.42|14.20|14.91|0.11|0.54|2.88|2.99|-11.32| +2452617|31683|9097|21669|1833048|6474|33053|10|145|23|81|72.51|122.54|121.31|0.00|9826.11|5873.31|9925.74|196.52|0.00|9826.11|10022.63|3952.80| +2452143|49413|2497|15069|576658|6247|49436|10|58|24|85|57.93|68.35|24.60|0.00|2091.00|4924.05|5809.75|62.73|0.00|2091.00|2153.73|-2833.05| +2452143|49413|4749|15069|576658|6247|49436|10|11|24|28|18.17|23.80|10.23|0.00|286.44|508.76|666.40|17.18|0.00|286.44|303.62|-222.32| +2452143|49413|6751|15069|576658|6247|49436|10|187|24|82|7.48|12.34|2.59|0.00|212.38|613.36|1011.88|0.00|0.00|212.38|212.38|-400.98| +2452143|49413|6043|15069|576658|6247|49436|10|280|24|1|33.41|59.80|29.30|0.00|29.30|33.41|59.80|0.00|0.00|29.30|29.30|-4.11| +2452143|49413|13303|15069|576658|6247|49436|10|101|24|97|94.48|172.89|157.32|0.00|15260.04|9164.56|16770.33|152.60|0.00|15260.04|15412.64|6095.48| +2452143|49413|7375|15069|576658|6247|49436|10|64|24|97|78.55|146.88|73.44|0.00|7123.68|7619.35|14247.36|356.18|0.00|7123.68|7479.86|-495.67| +2452143|49413|10265|15069|576658|6247|49436|10|231|24|93|12.03|23.09|11.31|0.00|1051.83|1118.79|2147.37|42.07|0.00|1051.83|1093.90|-66.96| +2452143|49413|1795|15069|576658|6247|49436|10|264|24|21|76.67|138.00|113.16|0.00|2376.36|1610.07|2898.00|190.10|0.00|2376.36|2566.46|766.29| +2452143|49413|11551|15069|576658|6247|49436|10|17|24|48|30.80|56.05|0.56|0.00|26.88|1478.40|2690.40|1.88|0.00|26.88|28.76|-1451.52| +2452143|49413|5653|15069|576658|6247|49436|10|119|24|92|64.99|67.58|27.03|223.80|2486.76|5979.08|6217.36|181.03|223.80|2262.96|2443.99|-3716.12| +2452143|49413|407|15069|576658|6247|49436|10|56|24|53|98.05|104.91|65.04|0.00|3447.12|5196.65|5560.23|241.29|0.00|3447.12|3688.41|-1749.53| +2452143|49413|1389|15069|576658|6247|49436|10|269|24|72|60.01|118.81|81.97|0.00|5901.84|4320.72|8554.32|59.01|0.00|5901.84|5960.85|1581.12| +2452143|49413|3103|15069|576658|6247|49436|10|235|24|73|44.96|56.20|12.92|0.00|943.16|3282.08|4102.60|47.15|0.00|943.16|990.31|-2338.92| +2452143|49413|4873|15069|576658|6247|49436|10|103|24|41|40.34|73.82|3.69|0.00|151.29|1653.94|3026.62|13.61|0.00|151.29|164.90|-1502.65| +2452143|49413|16483|15069|576658|6247|49436|10|291|24|89|13.84|14.53|14.09|100.32|1254.01|1231.76|1293.17|57.68|100.32|1153.69|1211.37|-78.07| +2452143|49413|4425|15069|576658|6247|49436|10|218|24|57|29.31|39.86|5.18|0.00|295.26|1670.67|2272.02|0.00|0.00|295.26|295.26|-1375.41| +2450920|36458|7874|70838|608693|5395|8050|10|243|25|65|2.75|4.26|2.34|31.94|152.10|178.75|276.90|7.20|31.94|120.16|127.36|-58.59| +2450920|36458|9472|70838|608693|5395|8050|10|148|25|48|4.97|7.70|2.69|0.00|129.12|238.56|369.60|7.74|0.00|129.12|136.86|-109.44| +2450920|36458|4880|70838|608693|5395|8050|10|189|25|29|15.35|19.18|0.57|13.88|16.53|445.15|556.22|0.00|13.88|2.65|2.65|-442.50| +2450920|36458|6955|70838|608693|5395|8050|10|150|25|40|87.12|107.15|40.71|0.00|1628.40|3484.80|4286.00|113.98|0.00|1628.40|1742.38|-1856.40| +2450920|36458|2150|70838|608693|5395|8050|10|270|25|100|67.24|86.73|55.50|0.00|5550.00|6724.00|8673.00|388.50|0.00|5550.00|5938.50|-1174.00| +2450920||14635|||||10||25|68|45.79|50.82|19.81|||3113.72|3455.76|80.82||||| +2450920|36458|14488|70838|608693|5395|8050|10|243|25|26|68.17|94.75|62.53|0.00|1625.78|1772.42|2463.50|65.03|0.00|1625.78|1690.81|-146.64| +2450920|36458|3454|70838|608693|5395|8050|10|9|25|100|1.61|2.83|0.79|73.47|79.00|161.00|283.00|0.05|73.47|5.53|5.58|-155.47| +2450920|36458|10159|70838|608693|5395|8050|10|110|25|24|76.64|96.56|5.79|84.76|138.96|1839.36|2317.44|0.00|84.76|54.20|54.20|-1785.16| +2450920|36458|17752|70838|608693|5395|8050|10|142|25|55|11.49|18.38|11.39|0.00|626.45|631.95|1010.90|0.00|0.00|626.45|626.45|-5.50| +2450920|36458|1333|70838|608693|5395|8050|10|93|25|55|30.82|51.46|26.75|0.00|1471.25|1695.10|2830.30|102.98|0.00|1471.25|1574.23|-223.85| +2450920|36458|17000|70838|608693|5395|8050|10|68|25|40|89.34|173.31|69.32|0.00|2772.80|3573.60|6932.40|83.18|0.00|2772.80|2855.98|-800.80| +2450920|36458|5954||608693|||10|220|25|34|76.57||20.67||702.78|2603.38||||702.78|709.80|-1900.60| +2450920|36458|2608|70838|608693|5395|8050|10|129|25|76|87.75|106.17|46.71|0.00|3549.96|6669.00|8068.92|177.49|0.00|3549.96|3727.45|-3119.04| +2452253|63557|11895|15005|179569|1380|23944|10|209|26|77|36.85|60.06|13.81|0.00|1063.37|2837.45|4624.62|95.70|0.00|1063.37|1159.07|-1774.08| +2452253|63557|15039|15005|179569|1380|23944|10|265|26|34|22.65|42.58|16.60|0.00|564.40|770.10|1447.72|50.79|0.00|564.40|615.19|-205.70| +2452253|63557|5721|15005|179569|1380|23944|10|92|26|81|74.96|95.19|82.81|0.00|6707.61|6071.76|7710.39|335.38|0.00|6707.61|7042.99|635.85| +2452253|63557|5385|15005|179569|1380|23944|10|59|26|97|51.89|86.65|40.72|0.00|3949.84|5033.33|8405.05|0.00|0.00|3949.84|3949.84|-1083.49| +2452253|63557|12851|15005|179569|1380|23944|10|217|26|56|14.02|26.63|4.26|0.00|238.56|785.12|1491.28|19.08|0.00|238.56|257.64|-546.56| +2452253|63557|6647|15005|179569|1380|23944|10|85|26|64|57.04|91.83|0.00|0.00|0.00|3650.56|5877.12|0.00|0.00|0.00|0.00|-3650.56| +2452253|63557|5053|15005|179569|1380|23944|10|200|26|4|19.63|33.56|22.14|0.00|88.56|78.52|134.24|7.08|0.00|88.56|95.64|10.04| +2452253|63557|7337|15005|179569|1380|23944|10|292|26|45|37.59|50.74|38.05|0.00|1712.25|1691.55|2283.30|0.00|0.00|1712.25|1712.25|20.70| +2452253|63557|1989|15005|179569|1380|23944|10|199|26|26|83.31|99.97|36.98|0.00|961.48|2166.06|2599.22|67.30|0.00|961.48|1028.78|-1204.58| +|63557|9679|15005|||23944||114|26|18|77.54|144.22||||1395.72|2595.96|||311.40|326.97|-1084.32| +2451942|67151|12237|32780|1342441|102|48163|1|6|27|89|73.90|138.19|103.64|0.00|9223.96|6577.10|12298.91|737.91|0.00|9223.96|9961.87|2646.86| +2451942|67151|2671|32780|1342441|102|48163|1|275|27|92|4.35|5.17|4.13|0.00|379.96|400.20|475.64|15.19|0.00|379.96|395.15|-20.24| +2451942|67151|15025|32780|1342441|102|48163|1|152|27|26|35.56|37.69|12.06|0.00|313.56|924.56|979.94|28.22|0.00|313.56|341.78|-611.00| +2451942|67151|2137|32780|1342441|102|48163|1|11|27|96|3.07|5.12|4.55|0.00|436.80|294.72|491.52|0.00|0.00|436.80|436.80|142.08| +2451942|67151|16029|32780|1342441|102|48163|1|243|27|59|2.11|3.22|3.22|0.00|189.98|124.49|189.98|17.09|0.00|189.98|207.07|65.49| +2451942|67151|7139|32780|1342441|102|48163|1|156|27|59|6.17|10.11|6.26|0.00|369.34|364.03|596.49|18.46|0.00|369.34|387.80|5.31| +2451942|67151|11441|32780|1342441|102|48163|1|63|27|15|14.45|27.31|2.18|0.00|32.70|216.75|409.65|2.61|0.00|32.70|35.31|-184.05| +2451942|67151|10757|32780|1342441|102|48163|1|142|27|15|8.70|12.18|3.89|0.00|58.35|130.50|182.70|4.66|0.00|58.35|63.01|-72.15| +2451942|67151|1305|32780|1342441|102|48163|1|255|27|44|8.35|13.86|11.91|0.00|524.04|367.40|609.84|0.00|0.00|524.04|524.04|156.64| +2451942|67151|12749|32780|1342441|102|48163|1|174|27|31|80.27|115.58|85.52|0.00|2651.12|2488.37|3582.98|106.04|0.00|2651.12|2757.16|162.75| +2451942|67151|11509|32780|1342441|102|48163|1|219|27|65|80.34|138.98|50.03|0.00|3251.95|5222.10|9033.70|195.11|0.00|3251.95|3447.06|-1970.15| +2451942|67151|16419|32780|1342441|102|48163|1|125|27|65|80.10|155.39|113.43|0.00|7372.95|5206.50|10100.35|516.10|0.00|7372.95|7889.05|2166.45| +2451942|67151|13885|32780|1342441|102|48163|1|103|27|66|40.62|53.61|43.42|0.00|2865.72|2680.92|3538.26|57.31|0.00|2865.72|2923.03|184.80| +2451942|67151|8167|32780|1342441|102|48163|1|214|27|28|38.83|69.89|37.74|0.00|1056.72|1087.24|1956.92|21.13|0.00|1056.72|1077.85|-30.52| +2451942|67151|5831|32780|1342441|102|48163|1|196|27|61|8.79|13.62|2.72|0.00|165.92|536.19|830.82|4.97|0.00|165.92|170.89|-370.27| +2451942|67151|16767|32780|1342441|102|48163|1|241|27|60|68.33|73.11|28.51|0.00|1710.60|4099.80|4386.60|34.21|0.00|1710.60|1744.81|-2389.20| +2452125|68718|6411|20087|1610670|453|31836|4|268|28|3|67.26|112.99|76.83|0.00|230.49|201.78|338.97|4.60|0.00|230.49|235.09|28.71| +2452125|68718|2817|20087|1610670|453|31836|4|100|28|8|98.75|190.58|83.85|429.31|670.80|790.00|1524.64|0.00|429.31|241.49|241.49|-548.51| +2452125|68718|8407|20087|1610670|453|31836|4|286|28|14|95.01|132.06|15.84|0.00|221.76|1330.14|1848.84|15.52|0.00|221.76|237.28|-1108.38| +2452125|68718|4483|20087|1610670|453|31836|4|285|28|78|73.90|141.14|117.14|0.00|9136.92|5764.20|11008.92|456.84|0.00|9136.92|9593.76|3372.72| +2452125|68718|7965|20087|1610670|453|31836|4|275|28|93|77.74|136.04|53.05|296.01|4933.65|7229.82|12651.72|0.00|296.01|4637.64|4637.64|-2592.18| +2452125|68718|2967|20087|1610670|453|31836|4|227|28|29|47.87|52.17|3.13|0.00|90.77|1388.23|1512.93|0.90|0.00|90.77|91.67|-1297.46| +2452125|68718|1807|20087|1610670|453|31836|4|192|28|98|78.91|133.35|132.01|0.00|12936.98|7733.18|13068.30|776.21|0.00|12936.98|13713.19|5203.80| +2452125|68718|5437|20087|1610670|453|31836|4|204|28|15|7.49|8.38|7.20|0.00|108.00|112.35|125.70|6.48|0.00|108.00|114.48|-4.35| +2452125|68718|10295|20087|1610670|453|31836|4|231|28|13|31.83|41.37|7.44|0.00|96.72|413.79|537.81|8.70|0.00|96.72|105.42|-317.07| +2452125|68718|8043|20087|1610670|453|31836|4|196|28|58|60.26|69.90|47.53|0.00|2756.74|3495.08|4054.20|137.83|0.00|2756.74|2894.57|-738.34| +2451121|45001|7654|10631|1267519|54|22989|7|159|29|20|98.14|171.74|171.74|1270.87|3434.80|1962.80|3434.80|0.00|1270.87|2163.93|2163.93|201.13| +2451121|45001|12223|10631|1267519|54|22989|7|56|29|59|12.89|14.05|0.84|33.20|49.56|760.51|828.95|1.47|33.20|16.36|17.83|-744.15| +2451121|45001|15085|10631|||22989|7||29|40|||2.93|0.00|117.20|||4.68|0.00|117.20||-297.60| +2451121|45001|3740|10631|1267519|54|22989|7|110|29|5|90.13|104.55|21.95|0.00|109.75|450.65|522.75|9.87|0.00|109.75|119.62|-340.90| +2451121||1363|||54|22989|7||29|75||97.65|3.90|0.00|292.50||7323.75||0.00|292.50|310.05|| +2451121|45001|12068|10631|1267519|54|22989|7|188|29|37|80.75|95.28|1.90|0.00|70.30|2987.75|3525.36|6.32|0.00|70.30|76.62|-2917.45| +2451121|45001|13340|10631|1267519|54|22989|7|142|29|21|40.50|49.00|29.89|0.00|627.69|850.50|1029.00|50.21|0.00|627.69|677.90|-222.81| +2451121|45001|2930|10631|1267519|54|22989|7|80|29|23|64.78|108.18|27.04|0.00|621.92|1489.94|2488.14|55.97|0.00|621.92|677.89|-868.02| +2451121|45001|15782|10631|1267519|54|22989|7|48|29|53|57.11|98.22|10.80|0.00|572.40|3026.83|5205.66|0.00|0.00|572.40|572.40|-2454.43| +2451121|45001|20|10631|1267519|54|22989|7|240|29|18|66.26|83.48|4.17|0.00|75.06|1192.68|1502.64|3.75|0.00|75.06|78.81|-1117.62| +2451121|45001|15626|10631|||||266|29||17.61||||467.40|1003.77|1374.84|9.34|||476.74|-536.37| +2451121|45001|9458|10631|1267519|54|22989|7|169|29|33|52.33|93.14|88.48|0.00|2919.84|1726.89|3073.62|29.19|0.00|2919.84|2949.03|1192.95| +2451121|45001|10795|10631|1267519|54|22989|7|32|29|33|68.24|116.69|57.17|1056.50|1886.61|2251.92|3850.77|66.40|1056.50|830.11|896.51|-1421.81| +2451121||13693|||||||29||95.63|184.56||0.00|0.00|4398.98|8489.76||0.00|||-4398.98| +2451390|52042|8666|45937|499127|7164|13554|7|36|30|21|64.00|81.28|29.26|0.00|614.46|1344.00|1706.88|36.86|0.00|614.46|651.32|-729.54| +2451390|52042|12572|45937|499127|7164|13554|7|270|30|33|61.96|67.53|14.85|210.72|490.05|2044.68|2228.49|19.55|210.72|279.33|298.88|-1765.35| +2451390|52042|12670|45937|499127|7164|13554|7|62|30|20|6.44|10.23|0.51|0.00|10.20|128.80|204.60|0.71|0.00|10.20|10.91|-118.60| +2451390|52042|3506|45937|499127|7164|13554|7|261|30|15|16.31|27.40|10.13|0.00|151.95|244.65|411.00|12.15|0.00|151.95|164.10|-92.70| +2451390|52042|6752|45937|499127|7164|13554|7|189|30|69|40.80|57.12|19.99|0.00|1379.31|2815.20|3941.28|0.00|0.00|1379.31|1379.31|-1435.89| +2451390|52042|5431|45937|499127|7164|13554|7|150|30|77|52.37|82.22|79.75|2701.93|6140.75|4032.49|6330.94|103.16|2701.93|3438.82|3541.98|-593.67| +2451390|52042|7870|45937|499127|7164|13554|7|256|30|7|4.51|6.85|6.02|18.12|42.14|31.57|47.95|0.00|18.12|24.02|24.02|-7.55| +2451390|52042|13579|45937|499127|7164|13554|7|230|30|75|62.71|77.76|9.33|0.00|699.75|4703.25|5832.00|27.99|0.00|699.75|727.74|-4003.50| +2451390|52042|3928|45937|499127|7164|13554|7|4|30|87|27.01|52.66|29.48|0.00|2564.76|2349.87|4581.42|179.53|0.00|2564.76|2744.29|214.89| +2451390|52042|1951|45937|499127|7164|13554|7|16|30|59|17.14|24.68|21.22|813.78|1251.98|1011.26|1456.12|35.05|813.78|438.20|473.25|-573.06| +2451390|52042|17348|45937|499127|7164|13554|7|57|30|62|88.74|169.49|74.57|1248.30|4623.34|5501.88|10508.38|101.25|1248.30|3375.04|3476.29|-2126.84| +2451390|52042|217|45937|499127|7164|13554|7|51|30|91|52.03|95.73|9.57|365.76|870.87|4734.73|8711.43|35.35|365.76|505.11|540.46|-4229.62| +2451390|52042|14848|45937|499127|7164|13554|7|78|30|62|64.03|103.08|41.23|1150.31|2556.26|3969.86|6390.96|28.11|1150.31|1405.95|1434.06|-2563.91| +2451390|52042|17875|45937|499127|7164|13554|7|90|30|78|2.91|4.56|0.41|0.00|31.98|226.98|355.68|0.95|0.00|31.98|32.93|-195.00| +2451390|52042|3238|45937|499127|7164|13554|7|148|30|16|9.84|15.84|8.55|0.00|136.80|157.44|253.44|1.36|0.00|136.80|138.16|-20.64| +2452187|71970|7513|3748|596752|4156|8760|7|113|31|82|44.95|72.36|42.69|0.00|3500.58|3685.90|5933.52|140.02|0.00|3500.58|3640.60|-185.32| +2452187|71970|14575|3748|596752|4156|8760|7|134|31|97|59.90|119.80|112.61|0.00|10923.17|5810.30|11620.60|218.46|0.00|10923.17|11141.63|5112.87| +2452187|71970|6617|3748|596752|4156|8760|7|75|31|11|86.78|91.98|57.94|0.00|637.34|954.58|1011.78|25.49|0.00|637.34|662.83|-317.24| +2452187|71970|15341|3748|596752|4156|8760|7|265|31|31|16.15|31.16|29.60|0.00|917.60|500.65|965.96|18.35|0.00|917.60|935.95|416.95| +2452187|71970|14727|3748|596752|4156|8760|7|134|31|41|48.10|53.87|8.08|0.00|331.28|1972.10|2208.67|29.81|0.00|331.28|361.09|-1640.82| +2452187|71970|913|3748|596752|4156|8760|7|133|31|54|79.11|93.34|59.73|2386.81|3225.42|4271.94|5040.36|67.08|2386.81|838.61|905.69|-3433.33| +2452187|71970|15411|3748|596752|4156|8760|7|60|31|53|47.64|61.45|30.11|973.45|1595.83|2524.92|3256.85|12.44|973.45|622.38|634.82|-1902.54| +2452187|71970|4963|3748|596752|4156|8760|7|103|31|67|56.37|59.18|37.28|1873.32|2497.76|3776.79|3965.06|0.00|1873.32|624.44|624.44|-3152.35| +2452187|71970|16251|3748|596752|4156|8760|7|65|31|51|91.49|182.06|138.36|0.00|7056.36|4665.99|9285.06|211.69|0.00|7056.36|7268.05|2390.37| +2452187|71970|6917|3748|596752|4156|8760|7|46|31|4|49.76|94.04|51.72|0.00|206.88|199.04|376.16|14.48|0.00|206.88|221.36|7.84| +2452187|71970|11739|3748|596752|4156|8760|7|83|31|95|6.99|11.32|5.09|0.00|483.55|664.05|1075.40|43.51|0.00|483.55|527.06|-180.50| +2451840|45877|4339|90120|1101094|6468|1395|4|107|32|6|88.27|139.46|26.49|0.00|158.94|529.62|836.76|7.94|0.00|158.94|166.88|-370.68| +2451840|45877|9683||1101094|6468|1395|4||32|26||149.26|2.98||77.48|2425.54||1.54||77.48||-2348.06| +2451840|45877|12134|90120|1101094|6468|1395|4|212|32|21|51.01|63.25|30.99|0.00|650.79|1071.21|1328.25|52.06|0.00|650.79|702.85|-420.42| +2451840||4537|90120||||4|202|32|22|65.72|69.00|57.27|0.00|1259.94||1518.00||0.00|1259.94||-185.90| +2451840|45877|4808|||6468||||32||57.01||||69.72|798.14|1396.64|||69.72|73.90|| +2451840|45877|5798|90120|1101094|6468|1395|4|40|32|87|46.23|91.99|55.19|4417.40|4801.53|4022.01|8003.13|11.52|4417.40|384.13|395.65|-3637.88| +2451840|45877|17659|90120|1101094|6468|1395|4|172|32|51|23.88|31.04|29.79|0.00|1519.29|1217.88|1583.04|45.57|0.00|1519.29|1564.86|301.41| +2451840|45877|15001|90120|1101094|6468|1395|4|181|32|57|30.07|54.42|23.94|0.00|1364.58|1713.99|3101.94|13.64|0.00|1364.58|1378.22|-349.41| +2451840|45877|11005|90120|1101094|6468|1395|4|174|32|46|51.48|79.27|72.92|1207.55|3354.32|2368.08|3646.42|21.46|1207.55|2146.77|2168.23|-221.31| +2451840|45877|15644|90120|1101094|6468|1395|4|123|32|34|80.54|131.28|89.27|0.00|3035.18|2738.36|4463.52|91.05|0.00|3035.18|3126.23|296.82| +2451840|45877|16421|90120|1101094|6468|1395|4|77|32|74|89.89|160.90|152.85|0.00|11310.90|6651.86|11906.60|113.10|0.00|11310.90|11424.00|4659.04| +2451840|45877|2095|90120|1101094|6468|1395|4|282|32|34|89.33|151.86|66.81|0.00|2271.54|3037.22|5163.24|0.00|0.00|2271.54|2271.54|-765.68| +2451840|45877|7547|90120|1101094|6468|1395|4|194|32|24|43.33|68.46|34.23|0.00|821.52|1039.92|1643.04|16.43|0.00|821.52|837.95|-218.40| +2451840|45877|2887|90120|1101094|6468|1395|4|108|32|8|48.71|96.93|32.95|0.00|263.60|389.68|775.44|15.81|0.00|263.60|279.41|-126.08| +2451840|45877|1115|90120|1101094|6468|1395|4|275|32|61|97.03|172.71|63.90|0.00|3897.90|5918.83|10535.31|194.89|0.00|3897.90|4092.79|-2020.93| +2451840|45877|11348|90120|1101094|6468|1395|4|274|32|41|55.14|57.34|51.03|753.20|2092.23|2260.74|2350.94|107.12|753.20|1339.03|1446.15|-921.71| +2451110|42673|7300|26724|445739|1201|4616|4|203|33|3|3.13|4.00|2.48|0.00|7.44|9.39|12.00|0.14|0.00|7.44|7.58|-1.95| +2451110|42673|15478|26724|445739|1201|4616|4|273|33|22|95.96|149.69|53.88|0.00|1185.36|2111.12|3293.18|82.97|0.00|1185.36|1268.33|-925.76| +2451110|42673|9649|26724|445739|1201|4616|4|233|33|36|18.91|31.01|22.63|0.00|814.68|680.76|1116.36|57.02|0.00|814.68|871.70|133.92| +2451110|42673|12964|26724|445739|1201|4616|4|266|33|80|83.86|114.04|25.08|0.00|2006.40|6708.80|9123.20|180.57|0.00|2006.40|2186.97|-4702.40| +2451110|42673|14692|26724|445739|1201|4616|4|190|33|93|56.78|107.88|90.61|0.00|8426.73|5280.54|10032.84|252.80|0.00|8426.73|8679.53|3146.19| +2451110|42673|12760|26724|445739|1201|4616|4|256|33|54|48.97|55.33|39.83|1161.44|2150.82|2644.38|2987.82|39.57|1161.44|989.38|1028.95|-1655.00| +2451110|42673|11119|26724|445739|1201|4616|4|242|33|92|3.49|3.69|1.95|0.00|179.40|321.08|339.48|7.17|0.00|179.40|186.57|-141.68| +2451110|42673|11756|26724|445739|1201|4616|4|121|33|26|58.87|89.48|57.26|0.00|1488.76|1530.62|2326.48|89.32|0.00|1488.76|1578.08|-41.86| +2451110|42673|14125|26724|445739|1201|4616|4|206|33|66|44.03|46.23|16.64|109.82|1098.24|2905.98|3051.18|19.76|109.82|988.42|1008.18|-1917.56| +2451110|42673|4798|26724|445739|1201|4616|4|178|33|27|28.56|37.98|13.67|350.63|369.09|771.12|1025.46|0.00|350.63|18.46|18.46|-752.66| +2451110|42673|14158|26724|445739|1201|4616|4|26|33|82|48.07|74.50|23.84|0.00|1954.88|3941.74|6109.00|0.00|0.00|1954.88|1954.88|-1986.86| +2451110|42673|12643|26724|445739|1201|4616|4|77|33|89|35.74|46.10|45.63|0.00|4061.07|3180.86|4102.90|81.22|0.00|4061.07|4142.29|880.21| +2451110|42673|10376|26724|445739|1201|4616|4|296|33|21|55.09|110.18|15.42|0.00|323.82|1156.89|2313.78|29.14|0.00|323.82|352.96|-833.07| +2451342|71555|9325|47971|1238972|6773|30574|7|173|34|3|92.35|109.89|48.35|37.71|145.05|277.05|329.67|6.44|37.71|107.34|113.78|-169.71| +2451342|71555|9766|47971|1238972|6773|30574|7|86|34|83|68.57|103.54|70.40|0.00|5843.20|5691.31|8593.82|467.45|0.00|5843.20|6310.65|151.89| +2451342|71555|12512|47971|1238972|6773|30574|7|63|34|60|40.48|78.93|22.10|0.00|1326.00|2428.80|4735.80|39.78|0.00|1326.00|1365.78|-1102.80| +2451342|71555|1717|47971|1238972|6773|30574|7|289|34|53|99.68|128.58|83.57|0.00|4429.21|5283.04|6814.74|265.75|0.00|4429.21|4694.96|-853.83| +2451342|71555|6193|47971|1238972|6773|30574|7|13|34|61|54.55|102.55|34.86|276.43|2126.46|3327.55|6255.55|166.50|276.43|1850.03|2016.53|-1477.52| +2451342||12290|47971|1238972|6773||7||34|53|88.61|108.99|||||5776.47|60.03||750.48||-3945.85| +2451342|71555|1526|47971|1238972|6773|30574|7|237|34|91|78.12|90.61|90.61|0.00|8245.51|7108.92|8245.51|82.45|0.00|8245.51|8327.96|1136.59| +2451342|71555|12016|47971|1238972|6773|30574|7|276|34|42|42.44|69.17|38.73|0.00|1626.66|1782.48|2905.14|146.39|0.00|1626.66|1773.05|-155.82| +2451342|71555|16324|47971|1238972|6773|30574|7|273|34|30|37.27|57.39|16.06|0.00|481.80|1118.10|1721.70|38.54|0.00|481.80|520.34|-636.30| +2451342|71555|4118|47971|1238972|6773|30574|7|121|34|88|38.38|74.84|71.84|0.00|6321.92|3377.44|6585.92|126.43|0.00|6321.92|6448.35|2944.48| +2451342|71555|5197|47971|1238972|6773|30574|7|190|34|63|13.50|26.46|17.19|0.00|1082.97|850.50|1666.98|32.48|0.00|1082.97|1115.45|232.47| +2451342|71555|2312|47971|1238972|6773|30574|7|231|34|6|51.40|78.64|20.44|0.00|122.64|308.40|471.84|7.35|0.00|122.64|129.99|-185.76| +||13814|47971||6773|30574|7|286|34|20|22.82||6.74||134.80|||||||| +2451342|71555|5449|47971|1238972|6773|30574|7|208|34|9|21.24|28.03|4.48|0.00|40.32|191.16|252.27|3.62|0.00|40.32|43.94|-150.84| +2452150|37970|15619|93958|616532|3106|34927|8|236|35|45|90.28|102.01|40.80|0.00|1836.00|4062.60|4590.45|36.72|0.00|1836.00|1872.72|-2226.60| +2452150|37970|17451|93958|616532|3106|34927|8|258|35|7|45.66|59.35|57.56|0.00|402.92|319.62|415.45|12.08|0.00|402.92|415.00|83.30| +2452150|37970|2377|93958|616532|3106|34927|8|2|35|52|98.03|140.18|2.80|90.27|145.60|5097.56|7289.36|0.55|90.27|55.33|55.88|-5042.23| +2452150|37970|5179|93958|616532|3106|34927|8|212|35|83|90.54|120.41|2.40|0.00|199.20|7514.82|9994.03|0.00|0.00|199.20|199.20|-7315.62| +2452150|37970|15757|93958|616532|3106|34927|8|68|35|9|14.83|27.43|0.27|0.00|2.43|133.47|246.87|0.19|0.00|2.43|2.62|-131.04| +2452150|37970|3667|93958|616532|3106|34927|8|250|35|97|59.31|60.49|48.99|3136.33|4752.03|5753.07|5867.53|32.31|3136.33|1615.70|1648.01|-4137.37| +2452150|37970|17341|93958|616532|3106|34927|8|287|35|92|59.48|116.58|68.78|0.00|6327.76|5472.16|10725.36|569.49|0.00|6327.76|6897.25|855.60| +2452150|37970|4325|93958|616532|3106|34927|8|25|35|56|67.43|126.76|115.35|0.00|6459.60|3776.08|7098.56|387.57|0.00|6459.60|6847.17|2683.52| +2452150|37970|11635|93958|616532|3106|34927|8|109|35|87|92.02|175.75|49.21|0.00|4281.27|8005.74|15290.25|342.50|0.00|4281.27|4623.77|-3724.47| +2452150|37970|14239|93958|616532|3106|34927|8|171|35|55|8.27|10.08|1.10|0.00|60.50|454.85|554.40|0.60|0.00|60.50|61.10|-394.35| +2452150|37970|11661||616532|3106|34927|8||35|81|||27.75|||||89.91|||2337.66|223.56| +2452150||411|93958|616532|3106||||35|51||73.41||0.00|1647.30|1940.04||131.78|0.00|||-292.74| +2452150|37970|17365|93958|616532|3106|34927|8|202|35|65|76.20|104.39|69.94|0.00|4546.10|4953.00|6785.35|363.68|0.00|4546.10|4909.78|-406.90| +2452274|74196|17659|72008|631970|1532|24694|8|286|36|91|44.83|73.52|25.73|0.00|2341.43|4079.53|6690.32|140.48|0.00|2341.43|2481.91|-1738.10| +2452274|74196|15001|72008|631970|1532|24694|8|159|36|54|16.24|26.14|22.21|0.00|1199.34|876.96|1411.56|95.94|0.00|1199.34|1295.28|322.38| +2452274|74196|11005|72008|631970|1532|24694|8|127|36|49|70.60|113.66|56.83|0.00|2784.67|3459.40|5569.34|83.54|0.00|2784.67|2868.21|-674.73| +2452274|74196|15645|72008|631970|1532|24694|8|136|36|23|32.35|34.93|4.54|0.00|104.42|744.05|803.39|7.30|0.00|104.42|111.72|-639.63| +2452274|74196|16421|72008|631970|1532|24694|8|144|36|25|69.67|104.50|77.33|0.00|1933.25|1741.75|2612.50|154.66|0.00|1933.25|2087.91|191.50| +2452274|74196|2095|72008|631970|1532|24694|8|147|36|43|91.17|165.01|57.75|0.00|2483.25|3920.31|7095.43|223.49|0.00|2483.25|2706.74|-1437.06| +2452274|74196|7547|72008|631970|1532|24694|8|240|36|46|8.04|15.83|10.60|0.00|487.60|369.84|728.18|39.00|0.00|487.60|526.60|117.76| +2452274|74196|2887|72008|631970|1532|24694|8|97|36|31|24.53|47.09|14.12|0.00|437.72|760.43|1459.79|26.26|0.00|437.72|463.98|-322.71| +2452274|74196|1115|72008|631970|1532|24694|8|45|36|80|11.13|12.46|6.72|198.91|537.60|890.40|996.80|10.16|198.91|338.69|348.85|-551.71| +2452274|74196|11349|72008|631970|1532|24694|8|65|36|80|58.17|94.23|15.07|0.00|1205.60|4653.60|7538.40|36.16|0.00|1205.60|1241.76|-3448.00| +2452274|74196|17561|72008|631970|1532|24694|8|115|36|16|82.46|157.49|17.32|0.00|277.12|1319.36|2519.84|13.85|0.00|277.12|290.97|-1042.24| +2452087|61560|10715|96252|1823714|4423|24685|10|116|37|52|89.22|133.83|111.07|0.00|5775.64|4639.44|6959.16|115.51|0.00|5775.64|5891.15|1136.20| +2452087|61560|16971|96252|1823714|4423|24685|10|236|37|12|53.97|85.27|15.34|0.00|184.08|647.64|1023.24|0.00|0.00|184.08|184.08|-463.56| +2452087|61560|10929|96252|1823714|4423|24685|10|240|37|88|65.45|80.50|47.49|0.00|4179.12|5759.60|7084.00|292.53|0.00|4179.12|4471.65|-1580.48| +2452087|61560|2997|96252|1823714|4423|24685|10|92|37|94|85.67|143.92|100.74|0.00|9469.56|8052.98|13528.48|568.17|0.00|9469.56|10037.73|1416.58| +2452087|61560|13171|96252|1823714|4423|24685|10|150|37|6|84.14|159.86|110.30|0.00|661.80|504.84|959.16|26.47|0.00|661.80|688.27|156.96| +2452087|61560|7283|96252|1823714|4423|24685|10|13|37|87|54.25|60.76|1.21|0.00|105.27|4719.75|5286.12|4.21|0.00|105.27|109.48|-4614.48| +2452087|61560|15337|96252|1823714|4423|24685|10|217|37|62|16.64|32.94|9.88|0.00|612.56|1031.68|2042.28|24.50|0.00|612.56|637.06|-419.12| +2452087|61560|17125||1823714||24685|10||37||||74.97|0.00||5369.06||173.93|0.00||4522.19|| +2451476|51103|9710|16409|414090|2171|25452|8|211|38|7|82.77|91.04|54.62|275.28|382.34|579.39|637.28|2.14|275.28|107.06|109.20|-472.33| +2451476|51103|6220|16409|414090|2171|25452|8|265|38|98|14.54|17.88|10.54|733.37|1032.92|1424.92|1752.24|5.99|733.37|299.55|305.54|-1125.37| +2451476|51103|7784|16409|414090|2171|25452|8|102|38|56|31.89|37.63|24.83|889.90|1390.48|1785.84|2107.28|0.00|889.90|500.58|500.58|-1285.26| +2451476|51103|10441|16409|414090|2171|25452|8|55|38|62|80.37|114.12|59.34|0.00|3679.08|4982.94|7075.44|36.79|0.00|3679.08|3715.87|-1303.86| +2451476|51103|2164|16409|414090|2171|25452|8|20|38|17|72.04|105.17|7.36|111.35|125.12|1224.68|1787.89|0.55|111.35|13.77|14.32|-1210.91| +||757|||2171||8||38|2||||0.00||190.42|239.92|6.38|0.00|91.16|97.54|| +2451476|51103|8870|16409|414090|2171|25452|8|33|38|15|46.69|85.90|10.30|0.00|154.50|700.35|1288.50|12.36|0.00|154.50|166.86|-545.85| +2451476|51103|3439|16409|414090|2171|25452|8|100|38|84|11.71|20.02|10.41|0.00|874.44|983.64|1681.68|43.72|0.00|874.44|918.16|-109.20| +2451476|51103|7018|16409|414090|2171|25452|8|115|38|15|69.78|133.27|129.27|0.00|1939.05|1046.70|1999.05|96.95|0.00|1939.05|2036.00|892.35| +2451476|51103|5113|16409|414090|2171|25452|8|236|38|73|50.59|84.99|79.89|0.00|5831.97|3693.07|6204.27|291.59|0.00|5831.97|6123.56|2138.90| +2451476|51103|4154|16409|414090|2171|25452|8|12|38|35|10.28|14.49|3.33|0.00|116.55|359.80|507.15|8.15|0.00|116.55|124.70|-243.25| +2451476|51103|15698|16409|414090|2171|25452|8|38|38|57|11.40|15.16|12.58|0.00|717.06|649.80|864.12|0.00|0.00|717.06|717.06|67.26| +2451874|48754|3476|58896|1112127|5358|8989|10|163|39|73|80.57|132.94|130.28|0.00|9510.44|5881.61|9704.62|95.10|0.00|9510.44|9605.54|3628.83| +2451874|48754|7697|58896|1112127|5358|8989|10|89|39|24|44.89|79.45|70.71|0.00|1697.04|1077.36|1906.80|152.73|0.00|1697.04|1849.77|619.68| +2451874|48754|4190|58896|1112127|5358|8989|10|288|39|86|35.56|37.33|29.49|0.00|2536.14|3058.16|3210.38|101.44|0.00|2536.14|2637.58|-522.02| +2451874|48754|386|58896|1112127|5358|8989|10|258|39|89|28.08|44.08|36.58|0.00|3255.62|2499.12|3923.12|260.44|0.00|3255.62|3516.06|756.50| +2451874|48754|8411|58896|1112127|5358|8989|10|226|39|96|35.69|54.60|50.23|0.00|4822.08|3426.24|5241.60|241.10|0.00|4822.08|5063.18|1395.84| +2451874|48754|7097|58896|1112127|5358|8989|10|186|39|78|33.10|51.96|27.53|0.00|2147.34|2581.80|4052.88|0.00|0.00|2147.34|2147.34|-434.46| +2451874|48754|1598|58896|1112127|5358|8989|10|254|39|64|44.63|54.44|15.24|0.00|975.36|2856.32|3484.16|48.76|0.00|975.36|1024.12|-1880.96| +2451874|48754|8078|58896|1112127|5358|8989|10|198|39|54|73.60|113.34|73.67|1352.58|3978.18|3974.40|6120.36|183.79|1352.58|2625.60|2809.39|-1348.80| +2451874|48754|4957|58896|1112127|5358|8989|10|72|39|24|16.10|22.37|10.96|0.00|263.04|386.40|536.88|5.26|0.00|263.04|268.30|-123.36| +2451874|48754|7604|58896|1112127|5358|8989|10|140|39|49|46.49|49.27|49.27|0.00|2414.23|2278.01|2414.23|48.28|0.00|2414.23|2462.51|136.22| +2451874|48754|15491|58896|1112127|5358|8989|10|34|39|54|3.20|4.80|2.01|103.11|108.54|172.80|259.20|0.16|103.11|5.43|5.59|-167.37| +2451874|48754|3943|58896|1112127|5358|8989|10|213|39|64|59.68|85.93|16.32|490.90|1044.48|3819.52|5499.52|16.60|490.90|553.58|570.18|-3265.94| +2451874|48754|5393|58896|1112127|5358|8989|10|45|39|98|58.75|67.56|20.94|0.00|2052.12|5757.50|6620.88|123.12|0.00|2052.12|2175.24|-3705.38| +2451874|48754|15625|58896|1112127|5358|8989|10|63|39|17|96.62|157.49|108.66|0.00|1847.22|1642.54|2677.33|36.94|0.00|1847.22|1884.16|204.68| +2451874|48754|7118|58896|1112127|5358|8989|10|161|39|67|68.99|101.41|31.43|0.00|2105.81|4622.33|6794.47|21.05|0.00|2105.81|2126.86|-2516.52| +2450925|62048|5098|66489|634761|2233|49705|4|90|40|35|33.91|41.70|22.93|497.58|802.55|1186.85|1459.50|21.34|497.58|304.97|326.31|-881.88| +2450925|62048|10094|66489|634761|2233|49705|4|64|40|80|8.63|9.92|8.63|669.68|690.40|690.40|793.60|1.24|669.68|20.72|21.96|-669.68| +2450925|62048|13658|66489|634761|2233|49705|4|21|40|53|56.42|112.27|33.68|0.00|1785.04|2990.26|5950.31|107.10|0.00|1785.04|1892.14|-1205.22| +2450925|62048|3985|66489|634761|2233|49705|4|280|40|63|22.94|37.16|0.00|0.00|0.00|1445.22|2341.08|0.00|0.00|0.00|0.00|-1445.22| +2450925|62048|5318|66489|634761|2233|49705|4|190|40|87|32.66|44.09|22.92|0.00|1994.04|2841.42|3835.83|119.64|0.00|1994.04|2113.68|-847.38| +||12050|66489||2233|49705|4|258|40||38.12||62.75|0.00|||3522.20|103.53|0.00||3554.78|| +2450925|62048|2854|66489|634761|2233|49705|4|196|40|71|10.62|19.85|2.77|33.43|196.67|754.02|1409.35|13.05|33.43|163.24|176.29|-590.78| +2450925|62048|16976|66489|634761|2233|49705|4|208|40|3|20.70|28.77|8.34|0.00|25.02|62.10|86.31|2.00|0.00|25.02|27.02|-37.08| +2450925|62048|3490|66489|634761|2233|49705|4|275|40|65|41.24|66.80|45.42|0.00|2952.30|2680.60|4342.00|206.66|0.00|2952.30|3158.96|271.70| +2450925|62048|10912|66489|634761|2233|49705|4|264|40|23|2.46|4.84|3.92|9.91|90.16|56.58|111.32|6.42|9.91|80.25|86.67|23.67| +2451094|69987|7402|87890|1540338|4786|12434|10|59|41|69|57.23|97.29|65.18|0.00|4497.42|3948.87|6713.01|224.87|0.00|4497.42|4722.29|548.55| +2451094|69987|3380|87890|1540338|4786|12434|10|195|41|88|14.11|15.37|13.06|0.00|1149.28|1241.68|1352.56|11.49|0.00|1149.28|1160.77|-92.40| +2451094|69987|15242|87890|1540338|4786|12434|10|141|41|87|48.25|59.34|20.17|0.00|1754.79|4197.75|5162.58|122.83|0.00|1754.79|1877.62|-2442.96| +2451094|69987|10|87890|1540338|4786|12434|10|275|41|50|54.36|102.74|78.08|0.00|3904.00|2718.00|5137.00|234.24|0.00|3904.00|4138.24|1186.00| +2451094|69987|10915|87890|1540338|4786|12434|10|256|41|81|91.63|174.09|19.14|201.54|1550.34|7422.03|14101.29|121.39|201.54|1348.80|1470.19|-6073.23| +2451094|69987|5566|87890|1540338|4786|12434|10|254|41|11|50.45|75.67|10.59|0.00|116.49|554.95|832.37|6.98|0.00|116.49|123.47|-438.46| +2451094|69987|9322|87890|1540338|4786|12434|10|276|41|8|59.40|87.31|82.94|0.00|663.52|475.20|698.48|19.90|0.00|663.52|683.42|188.32| +2451094|69987|16514|87890|1540338|4786|12434|10|285|41|20|5.05|5.20|5.09|78.38|101.80|101.00|104.00|0.46|78.38|23.42|23.88|-77.58| +2451094|69987|15328|87890|1540338|4786|12434|10|8|41|46|84.03|168.06|70.58|0.00|3246.68|3865.38|7730.76|227.26|0.00|3246.68|3473.94|-618.70| +2451094|69987|7603|87890|1540338|4786|12434|10|166|41|94|6.12|9.91|3.17|0.00|297.98|575.28|931.54|23.83|0.00|297.98|321.81|-277.30| +2451094|69987|6310|87890|1540338|4786|12434|10|202|41|90|60.10|63.70|17.83|978.86|1604.70|5409.00|5733.00|25.03|978.86|625.84|650.87|-4783.16| +2451094|69987|14788|87890|1540338|4786|12434|10|222|41|15|90.04|108.04|70.22|0.00|1053.30|1350.60|1620.60|31.59|0.00|1053.30|1084.89|-297.30| +2451094|69987|64|87890|1540338|4786|12434|10|286|41|29|27.18|52.72|4.21|120.86|122.09|788.22|1528.88|0.06|120.86|1.23|1.29|-786.99| +2450952|40089|2314|49423|170973|3276|11226|10|16|42|74|24.46|39.38|16.93|0.00|1252.82|1810.04|2914.12|87.69|0.00|1252.82|1340.51|-557.22| +2450952|40089|10016|49423|170973|3276|11226|10|76|42|57|12.47|13.96|1.39|0.00|79.23|710.79|795.72|4.75|0.00|79.23|83.98|-631.56| +2450952|40089|3346|49423|170973|3276|11226|10|191|42|74|29.72|48.74|36.55|0.00|2704.70|2199.28|3606.76|0.00|0.00|2704.70|2704.70|505.42| +2450952|40089|976|49423|170973|3276|11226|10|21|42|100|12.59|19.13|8.22|0.00|822.00|1259.00|1913.00|24.66|0.00|822.00|846.66|-437.00| +2450952|40089|10471|49423|170973|3276|11226|10|13|42|19|42.67|42.67|10.24|0.00|194.56|810.73|810.73|15.56|0.00|194.56|210.12|-616.17| +2450952|40089|6058|49423|170973|3276|11226|10|120|42|30|81.10|139.49|33.47|0.00|1004.10|2433.00|4184.70|50.20|0.00|1004.10|1054.30|-1428.90| +2450952|40089|12550|49423|170973|3276|11226|10|199|42|41|17.09|22.90|6.41|0.00|262.81|700.69|938.90|7.88|0.00|262.81|270.69|-437.88| +2450952|40089|2392|49423|170973|3276|11226|10|163|42|14|49.48|72.24|55.62|0.00|778.68|692.72|1011.36|31.14|0.00|778.68|809.82|85.96| +2450952|40089|15002|49423|170973|3276|11226|10|21|42|41|58.33|80.49|38.63|0.00|1583.83|2391.53|3300.09|126.70|0.00|1583.83|1710.53|-807.70| +2450952|40089|6134|49423|170973|3276|11226|10|146|42|92|18.91|33.09|23.82|0.00|2191.44|1739.72|3044.28|109.57|0.00|2191.44|2301.01|451.72| +2450952|40089|3613|49423|170973|3276|11226|10|8|42|30|56.33|65.34|56.84|528.61|1705.20|1689.90|1960.20|0.00|528.61|1176.59|1176.59|-513.31| +2450952|40089|9740|49423|170973|3276|11226|10|268|42|52|52.46|90.23|32.48|1621.40|1688.96|2727.92|4691.96|1.35|1621.40|67.56|68.91|-2660.36| +2450952|40089|619|49423|170973|3276|11226|10|55|42|69|56.85|80.15|28.85|0.00|1990.65|3922.65|5530.35|0.00|0.00|1990.65|1990.65|-1932.00| +2450952|40089|1436|49423|170973|3276|11226|10|7|42|94|54.21|81.31|36.58|0.00|3438.52|5095.74|7643.14|68.77|0.00|3438.52|3507.29|-1657.22| +2450952|40089|8462|49423|170973|3276|11226|10|241|42|23|27.88|33.45|7.02|24.21|161.46|641.24|769.35|0.00|24.21|137.25|137.25|-503.99| +2450952|40089|2602|49423|170973|3276|11226|10|96|42|30|55.77|71.38|63.52|0.00|1905.60|1673.10|2141.40|76.22|0.00|1905.60|1981.82|232.50| +2451878|68553|5852|80811|1454478|5392|12259|4|1|43|16|81.99|84.44|81.06|0.00|1296.96|1311.84|1351.04|51.87|0.00|1296.96|1348.83|-14.88| +2451878|68553|14669|80811|1454478|5392|12259|4|269|43|97|52.94|98.99|12.86|0.00|1247.42|5135.18|9602.03|62.37|0.00|1247.42|1309.79|-3887.76| +2451878|68553|3911|80811|1454478|5392|12259|4|55|43|26|21.75|36.97|34.01|0.00|884.26|565.50|961.22|61.89|0.00|884.26|946.15|318.76| +2451878|68553|2923|80811|1454478|5392|12259|4|274|43|16|82.12|136.31|80.42|0.00|1286.72|1313.92|2180.96|51.46|0.00|1286.72|1338.18|-27.20| +2451878|68553|3344|80811|1454478|5392|12259|4|47|43|22|88.77|109.18|82.97|0.00|1825.34|1952.94|2401.96|0.00|0.00|1825.34|1825.34|-127.60| +2451878|68553|11771|80811|1454478|5392|12259|4|173|43|30|41.13|46.88|41.25|420.75|1237.50|1233.90|1406.40|49.00|420.75|816.75|865.75|-417.15| +2451878|68553|5773|80811|1454478|5392|12259|4|220|43|85|66.42|87.67|77.14|0.00|6556.90|5645.70|7451.95|0.00|0.00|6556.90|6556.90|911.20| +2451878|68553|4691|80811|1454478|5392|12259|4|63|43|41|2.24|4.39|0.00|0.00|0.00|91.84|179.99|0.00|0.00|0.00|0.00|-91.84| +2451878|68553|4364|80811|1454478|5392|12259|4|266|43|77|82.92|106.96|14.97|0.00|1152.69|6384.84|8235.92|11.52|0.00|1152.69|1164.21|-5232.15| +2452274|44359|7527|53792|336661|4330|25999|7|110|44|67|90.05|124.26|93.19|0.00|6243.73|6033.35|8325.42|437.06|0.00|6243.73|6680.79|210.38| +2452274|44359|2351|53792|336661|4330|25999|7|233|44|56|55.53|96.06|35.54|1850.92|1990.24|3109.68|5379.36|11.14|1850.92|139.32|150.46|-2970.36| +2452274|44359|2623|53792|336661|4330|25999|7|2|44|18|39.17|74.03|65.14|0.00|1172.52|705.06|1332.54|82.07|0.00|1172.52|1254.59|467.46| +2452274|44359|9059|53792|336661|4330|25999|7|173|44|68|30.11|50.28|18.60|0.00|1264.80|2047.48|3419.04|50.59|0.00|1264.80|1315.39|-782.68| +2452274|44359|12341|53792|336661|4330|25999|7|127|44|20|82.28|124.24|73.30|0.00|1466.00|1645.60|2484.80|14.66|0.00|1466.00|1480.66|-179.60| +2452274|44359|14149|53792|336661|4330|25999|7|164|44|80|18.83|22.40|16.35|0.00|1308.00|1506.40|1792.00|78.48|0.00|1308.00|1386.48|-198.40| +2452274|44359|15803|53792|336661|4330|25999|7|295|44|81|43.81|83.23|4.99|0.00|404.19|3548.61|6741.63|12.12|0.00|404.19|416.31|-3144.42| +2452274|44359|16491|53792|336661|4330|25999|7|103|44|56|32.28|53.26|13.31|0.00|745.36|1807.68|2982.56|52.17|0.00|745.36|797.53|-1062.32| +2452274|44359|13331|53792|336661|4330|25999|7|240|44|98|3.53|6.67|0.20|0.00|19.60|345.94|653.66|0.00|0.00|19.60|19.60|-326.34| +2452274|44359|16909|53792|336661|4330|25999|7|288|44|61|92.15|148.36|45.99|0.00|2805.39|5621.15|9049.96|56.10|0.00|2805.39|2861.49|-2815.76| +2452274|44359|11707|53792|336661|4330|25999|7|155|44|83|85.49|126.52|41.75|138.61|3465.25|7095.67|10501.16|232.86|138.61|3326.64|3559.50|-3769.03| +2452274|44359|13449|53792|336661|4330|25999|7|71|44|45|50.83|86.91|86.04|0.00|3871.80|2287.35|3910.95|309.74|0.00|3871.80|4181.54|1584.45| +2452274|44359|16837|53792|336661|4330|25999|7|265|44|92|30.11|37.63|35.37|0.00|3254.04|2770.12|3461.96|260.32|0.00|3254.04|3514.36|483.92| +2452274|44359|7303|53792|336661|4330|25999|7|50|44|14|36.13|62.14|19.26|0.00|269.64|505.82|869.96|18.87|0.00|269.64|288.51|-236.18| +2452257|37558|10115|16606|596784|6209|42787|4|37|45|68|38.09|58.65|7.62|310.89|518.16|2590.12|3988.20|0.00|310.89|207.27|207.27|-2382.85| +2452257|37558|16349|16606|596784|6209|42787|4|213|45|6|94.59|150.39|99.25|0.00|595.50|567.54|902.34|0.00|0.00|595.50|595.50|27.96| +2452257|37558|11095|16606|596784|6209|42787|4|235|45|40|52.97|90.04|69.33|0.00|2773.20|2118.80|3601.60|249.58|0.00|2773.20|3022.78|654.40| +2452257|37558|1479|16606|596784|6209|42787|4|299|45|49|5.01|8.91|0.00|0.00|0.00|245.49|436.59|0.00|0.00|0.00|0.00|-245.49| +2452257|37558|7363|16606|596784|6209|42787|4|137|45|87|85.95|87.66|4.38|0.00|381.06|7477.65|7626.42|7.62|0.00|381.06|388.68|-7096.59| +2452257|37558|13133|16606|596784|6209|42787|4|170|45|46|85.87|126.22|68.15|0.00|3134.90|3950.02|5806.12|219.44|0.00|3134.90|3354.34|-815.12| +2452257|37558|5309|16606|596784|6209|42787|4|44|45|18|51.16|83.39|21.68|0.00|390.24|920.88|1501.02|31.21|0.00|390.24|421.45|-530.64| +2452257|37558|811|16606|596784|6209|42787|4|265|45|62|23.41|29.26|4.09|0.00|253.58|1451.42|1814.12|12.67|0.00|253.58|266.25|-1197.84| +2452257|37558|3265|16606|596784|6209|42787|4|246|45|98|27.12|40.13|31.70|0.00|3106.60|2657.76|3932.74|279.59|0.00|3106.60|3386.19|448.84| +2451299|47778|15595|34598|1011655|6312|14569|8|53|46|32|29.12|35.81|12.17|0.00|389.44|931.84|1145.92|0.00|0.00|389.44|389.44|-542.40| +2451299|47778|17384|34598|1011655|6312|14569|8|266|46|13|93.68|140.52|25.29|0.00|328.77|1217.84|1826.76|3.28|0.00|328.77|332.05|-889.07| +2451299|47778|17150|34598|1011655|6312|14569|8|221|46|57|71.68|115.40|79.62|0.00|4538.34|4085.76|6577.80|181.53|0.00|4538.34|4719.87|452.58| +2451299|47778|1960|34598|1011655|6312|14569|8|133|46|12|53.47|92.50|73.07|0.00|876.84|641.64|1110.00|26.30|0.00|876.84|903.14|235.20| +2451299|47778|7040|34598|1011655|6312|14569|8|279|46|33|90.87|97.23|25.27|0.00|833.91|2998.71|3208.59|33.35|0.00|833.91|867.26|-2164.80| +2451299|47778|16540|34598|1011655|6312|14569|8|240|46|30|54.36|73.92|69.48|0.00|2084.40|1630.80|2217.60|125.06|0.00|2084.40|2209.46|453.60| +2451299|47778|13042|34598|1011655|6312|14569|8|135|46|58|41.38|49.65|5.46|0.00|316.68|2400.04|2879.70|6.33|0.00|316.68|323.01|-2083.36| +2451299|47778|8065|34598||6312||8|120|46||43.04||||860.25||4099.50||||894.66|| +2451299|47778|3010|34598|1011655|6312|14569|8|86|46|67|66.87|83.58|58.50|0.00|3919.50|4480.29|5599.86|313.56|0.00|3919.50|4233.06|-560.79| +2451299|47778|11426|34598|1011655|6312|14569|8|221|46|72|53.81|99.01|42.57|0.00|3065.04|3874.32|7128.72|61.30|0.00|3065.04|3126.34|-809.28| +2451147|42794|3085|83903|534604|58|5159|4|268|47|51|63.55|104.85|53.47|490.85|2726.97|3241.05|5347.35|67.08|490.85|2236.12|2303.20|-1004.93| +2451147|42794|13477|83903|534604|58|5159|4|22|47|10|78.83|143.47|58.82|0.00|588.20|788.30|1434.70|11.76|0.00|588.20|599.96|-200.10| +2451147|42794|5468|83903|534604|58|5159|4|156|47|7|5.03|7.44|0.66|0.00|4.62|35.21|52.08|0.04|0.00|4.62|4.66|-30.59| +2451147||5305|83903|534604|||||47||84.70|102.48|95.30|0.00|||||0.00|8291.10||| +2451147|42794|1132|83903|534604|58|5159|4|144|47|66|53.46|79.65|63.72|0.00|4205.52|3528.36|5256.90|42.05|0.00|4205.52|4247.57|677.16| +2451147|42794|4774|83903|534604|58|5159|4|147|47|11|63.71|102.57|58.46|0.00|643.06|700.81|1128.27|38.58|0.00|643.06|681.64|-57.75| +2451147|42794|5008|83903|534604|58|5159|4|164|47|82|1.76|2.62|0.83|0.00|68.06|144.32|214.84|4.08|0.00|68.06|72.14|-76.26| +2451147|42794|254|83903|534604||5159||138|47|||||0.00|7444.36|||595.54|0.00||8039.90|623.70| +2451147|42794|3202|83903|534604|58|5159|4|205|47|7|26.06|49.77|28.36|0.00|198.52|182.42|348.39|3.97|0.00|198.52|202.49|16.10| +2451147|42794|1916|83903|534604|58|5159|4|150|47|71|47.62|49.04|43.64|0.00|3098.44|3381.02|3481.84|278.85|0.00|3098.44|3377.29|-282.58| +2451147|42794|481|83903|534604|58|5159|4|92|47|30|36.51|48.92|6.35|0.00|190.50|1095.30|1467.60|5.71|0.00|190.50|196.21|-904.80| +2451147|42794|9770|83903|534604|58|5159|4|154|47|33|69.12|133.40|45.35|0.00|1496.55|2280.96|4402.20|0.00|0.00|1496.55|1496.55|-784.41| +|42794|3878|||58||4||47||||111.69||||4582.40|||||| +2451147|42794|7214|83903|534604|58|5159|4|196|47|1|12.80|17.28|8.12|0.00|8.12|12.80|17.28|0.73|0.00|8.12|8.85|-4.68| +2451147|42794|13246|83903|534604|58|5159|4|139|47|47|11.71|16.51|0.16|6.16|7.52|550.37|775.97|0.00|6.16|1.36|1.36|-549.01| +2452606|71569|9199|21202|888343|6237|1331|8|246|48|88|37.89|49.63|27.79|0.00|2445.52|3334.32|4367.44|97.82|0.00|2445.52|2543.34|-888.80| +2452606|71569|12468|21202|888343|6237|1331|8|164|48|62|43.72|66.45|39.87|0.00|2471.94|2710.64|4119.90|222.47|0.00|2471.94|2694.41|-238.70| +2452606|71569|13153|21202|888343|6237|1331|8|187|48|74|34.26|42.48|5.94|0.00|439.56|2535.24|3143.52|8.79|0.00|439.56|448.35|-2095.68| +2452606|71569|2820|21202|888343|6237|1331|8|23|48|4|6.46|9.69|6.68|16.29|26.72|25.84|38.76|0.00|16.29|10.43|10.43|-15.41| +2452606||11259||||1331|8||48||||39.00|0.00|897.00||1520.53|35.88|0.00|||| +2452606|71569|17799|21202|888343|6237|1331|8|82|48|17|80.36|151.07|120.85|0.00|2054.45|1366.12|2568.19|41.08|0.00|2054.45|2095.53|688.33| +2452606|71569|2829|21202|888343|6237|1331|8|289|48|65|22.10|27.84|24.49|0.00|1591.85|1436.50|1809.60|31.83|0.00|1591.85|1623.68|155.35| +2452606|71569|5971|21202|888343|6237|1331|8|110|48|29|40.46|54.21|39.03|0.00|1131.87|1173.34|1572.09|90.54|0.00|1131.87|1222.41|-41.47| +2452606|71569|6684|21202|888343|6237|1331|8|117|48|44|20.22|40.03|39.22|0.00|1725.68|889.68|1761.32|120.79|0.00|1725.68|1846.47|836.00| +2452606|71569|4431|21202|888343|6237|1331|8|110|48|39|97.07|177.63|5.32|0.00|207.48|3785.73|6927.57|0.00|0.00|207.48|207.48|-3578.25| +2452606|71569|6085|21202|888343|6237|1331|8|41|48|1|58.13|102.30|82.86|0.00|82.86|58.13|102.30|2.48|0.00|82.86|85.34|24.73| +2452606|71569|1761|21202|888343|6237|1331|8|183|48|22|55.73|63.53|0.63|0.00|13.86|1226.06|1397.66|0.41|0.00|13.86|14.27|-1212.20| +2451774|39843|9049|79401|1604468|3582|4296|8|246|49|68|75.38|144.72|115.77|0.00|7872.36|5125.84|9840.96|551.06|0.00|7872.36|8423.42|2746.52| +2451774|39843|9553|79401|1604468|3582|4296|8|149|49|71|29.28|44.21|3.97|0.00|281.87|2078.88|3138.91|22.54|0.00|281.87|304.41|-1797.01| +2451774|39843|12737|79401|1604468|3582|4296|8|107|49|48|2.17|4.01|2.60|0.00|124.80|104.16|192.48|8.73|0.00|124.80|133.53|20.64| +2451774|39843|7141|79401|1604468|3582|4296|8|139|49|94|50.50|63.63|57.26|4521.24|5382.44|4747.00|5981.22|51.67|4521.24|861.20|912.87|-3885.80| +2451774|39843|749|79401|1604468|3582|4296|8|8|49|60|42.11|83.37|76.70|0.00|4602.00|2526.60|5002.20|0.00|0.00|4602.00|4602.00|2075.40| +2451774|39843|6805|79401|1604468|3582|4296|8|58|49|40|12.90|25.41|23.63|0.00|945.20|516.00|1016.40|56.71|0.00|945.20|1001.91|429.20| +2451774|39843|16361|79401|1604468|3582|4296|8|240|49|4|79.85|125.36|6.26|0.00|25.04|319.40|501.44|1.75|0.00|25.04|26.79|-294.36| +2451774|39843|2135|79401|1604468|3582|4296|8|103|49|4|15.80|16.59|7.46|0.00|29.84|63.20|66.36|1.79|0.00|29.84|31.63|-33.36| +2451774|39843|15155|79401|1604468|3582|4296|8|81|49|84|4.40|6.60|2.24|0.00|188.16|369.60|554.40|3.76|0.00|188.16|191.92|-181.44| +2451774|39843|5342|79401|1604468|3582|4296|8|89|49|69|46.41|46.87|41.71|1957.03|2877.99|3202.29|3234.03|27.62|1957.03|920.96|948.58|-2281.33| +2451774|39843|5852|79401|1604468|3582|4296|8|45|49|47|74.90|77.89|45.17|0.00|2122.99|3520.30|3660.83|21.22|0.00|2122.99|2144.21|-1397.31| +2451203|46470|1280|55849|1179944|5639|14534|4|231|50|69|8.66|14.80|1.62|0.00|111.78|597.54|1021.20|5.58|0.00|111.78|117.36|-485.76| +2451203|46470|16081|55849|1179944|5639|14534|4|206|50|82|99.55|124.43|2.48|0.00|203.36|8163.10|10203.26|14.23|0.00|203.36|217.59|-7959.74| +2451203|46470|1312|55849|1179944|5639|14534|4|8|50|30|25.84|49.35|9.87|0.00|296.10|775.20|1480.50|5.92|0.00|296.10|302.02|-479.10| +2451203|46470|7135|55849|1179944|5639|14534|4|199|50|69|12.68|21.42|19.70|1182.59|1359.30|874.92|1477.98|3.53|1182.59|176.71|180.24|-698.21| +2451203|46470|1984|55849|1179944|5639|14534|4|188|50|40|8.81|8.81|6.25|222.50|250.00|352.40|352.40|0.82|222.50|27.50|28.32|-324.90| +2451203|46470|5023|||||||50||16.24|||0.00|83.44||132.48|0.00|0.00|||-46.48| +2451203|46470|1909|55849|1179944|5639|14534|4|86|50|53|56.01|62.17|6.83|322.17|361.99|2968.53|3295.01|0.00|322.17|39.82|39.82|-2928.71| +2451203|46470|3097|55849|1179944|5639|14534|4|191|50|64|33.17|55.39|27.14|0.00|1736.96|2122.88|3544.96|156.32|0.00|1736.96|1893.28|-385.92| +2451693|47958|12625|83517|38885|1806|28327|10|169|51|53|97.27|176.05|116.19|0.00|6158.07|5155.31|9330.65|307.90|0.00|6158.07|6465.97|1002.76| +2451693|47958|3091|83517|38885|1806|28327|10|136|51|28|92.87|120.73|84.51|0.00|2366.28|2600.36|3380.44|23.66|0.00|2366.28|2389.94|-234.08| +2451693|47958|422|83517|38885|1806|28327|10|166|51|21|69.89|131.39|97.22|306.24|2041.62|1467.69|2759.19|52.06|306.24|1735.38|1787.44|267.69| +2451693|47958|5029|83517|38885|1806|28327|10|227|51|12|34.53|46.96|29.58|0.00|354.96|414.36|563.52|10.64|0.00|354.96|365.60|-59.40| +2451693|47958|17705|83517|38885|1806|28327|10|86|51|66|46.11|60.86|23.12|0.00|1525.92|3043.26|4016.76|91.55|0.00|1525.92|1617.47|-1517.34| +2451693||11234||38885|1806|28327|10||51|||113.10||0.00||1187.55|||0.00||1859.09|| +2451693|47958|17483|83517|38885|1806|28327|10|223|51|22|31.99|32.94|24.37|160.84|536.14|703.78|724.68|11.25|160.84|375.30|386.55|-328.48| +2451693|47958|8384|83517|38885|1806|28327|10|298|51|79|15.35|29.01|22.33|0.00|1764.07|1212.65|2291.79|158.76|0.00|1764.07|1922.83|551.42| +2451693|47958|6565|83517|38885|1806|28327|10|165|51|33|32.44|55.47|2.21|0.00|72.93|1070.52|1830.51|5.83|0.00|72.93|78.76|-997.59| +2451693|47958|5059|83517|38885|1806|28327|10|78|51|51|48.54|85.43|0.85|0.00|43.35|2475.54|4356.93|0.86|0.00|43.35|44.21|-2432.19| +2451693|47958|4687|83517|38885|1806|28327|10|129|51|6|93.02|97.67|9.76|0.00|58.56|558.12|586.02|5.27|0.00|58.56|63.83|-499.56| +2451693|47958|9311|83517|38885|1806|28327|10|77|51|90|39.48|53.69|7.51|0.00|675.90|3553.20|4832.10|47.31|0.00|675.90|723.21|-2877.30| +2451693|47958|13199|83517|38885|1806|28327|10|198|51|97|99.32|105.27|94.74|0.00|9189.78|9634.04|10211.19|459.48|0.00|9189.78|9649.26|-444.26| +2451693|47958|10133|83517|38885|1806|28327|10|246|51|54|46.71|51.38|34.42|55.76|1858.68|2522.34|2774.52|54.08|55.76|1802.92|1857.00|-719.42| +2451027|41589|7606|45831|354191|951|5264|7|30|52|45|42.51|72.26|3.61|0.00|162.45|1912.95|3251.70|0.00|0.00|162.45|162.45|-1750.50| +2451027|41589|15649|45831|354191|951|5264|7|25|52|29|22.86|29.71|16.04|0.00|465.16|662.94|861.59|13.95|0.00|465.16|479.11|-197.78| +2451027|41589|11488|45831|354191|951|5264|7|108|52|76|78.68|106.21|86.03|0.00|6538.28|5979.68|8071.96|130.76|0.00|6538.28|6669.04|558.60| +2451027|41589|2420|45831|354191|951|5264|7|94|52|90|22.31|35.24|18.67|0.00|1680.30|2007.90|3171.60|100.81|0.00|1680.30|1781.11|-327.60| +2451027|41589|16646|45831|354191|951|5264|7|165|52|48|95.82|132.23|70.08|773.68|3363.84|4599.36|6347.04|0.00|773.68|2590.16|2590.16|-2009.20| +2451027|41589|17456|45831|354191|951|5264|7|71|52|37|7.93|11.97|4.54|0.00|167.98|293.41|442.89|1.67|0.00|167.98|169.65|-125.43| +|41589|6098||354191||||45|52||4.83|7.63|7.40||||495.95|||481.00|481.00|| +2451027|41589|3334|45831|354191|951|5264|7|8|52|73|29.20|39.71|29.78|1260.88|2173.94|2131.60|2898.83|73.04|1260.88|913.06|986.10|-1218.54| +2451027|41589|17402|45831|354191|951|5264|7|242|52|91|81.94|139.29|123.96|0.00|11280.36|7456.54|12675.39|789.62|0.00|11280.36|12069.98|3823.82| +2451452|39338|16420|87672|1260894|5316|32985|4|117|53|14|36.72|63.15|56.20|0.00|786.80|514.08|884.10|15.73|0.00|786.80|802.53|272.72| +2451452|39338|2095|87672|1260894|5316|32985|4|209|53|62|23.98|32.61|28.69|0.00|1778.78|1486.76|2021.82|53.36|0.00|1778.78|1832.14|292.02| +2451452|39338|7546|87672|1260894|5316|32985|4|39|53|58|73.79|118.80|91.47|0.00|5305.26|4279.82|6890.40|424.42|0.00|5305.26|5729.68|1025.44| +2451452|39338|2887|87672|1260894|5316|32985|4|175|53|39|66.68|106.02|99.65|0.00|3886.35|2600.52|4134.78|349.77|0.00|3886.35|4236.12|1285.83| +2451452|39338|1114|87672|1260894|5316|32985|4|178|53|40|28.34|38.25|5.73|192.52|229.20|1133.60|1530.00|0.36|192.52|36.68|37.04|-1096.92| +2451452|39338|11348|87672|1260894|5316|32985|4|277|53|38|5.54|9.30|2.23|0.00|84.74|210.52|353.40|3.38|0.00|84.74|88.12|-125.78| +2451452|39338|17560|87672|1260894|5316|32985|4|218|53|15|46.39|68.19|62.73|592.79|940.95|695.85|1022.85|24.37|592.79|348.16|372.53|-347.69| +2451452|39338|17296|87672|1260894|5316|32985|4|194|53|43|21.40|29.10|0.58|0.00|24.94|920.20|1251.30|0.00|0.00|24.94|24.94|-895.26| +2451452|39338|13220|87672|1260894|5316|32985|4|135|53|76|27.93|49.15|25.55|0.00|1941.80|2122.68|3735.40|174.76|0.00|1941.80|2116.56|-180.88| +2451452|39338|13795|87672|1260894|5316|32985|4|260|53|38|93.96|177.58|17.75|0.00|674.50|3570.48|6748.04|26.98|0.00|674.50|701.48|-2895.98| +2451452|39338|16648|87672|1260894|5316|32985|4|208|53|79|55.29|95.09|9.50|570.38|750.50|4367.91|7512.11|1.80|570.38|180.12|181.92|-4187.79| +2451452|39338|15991|87672|1260894|5316|32985|4|39|53|37|77.75|87.08|27.86|0.00|1030.82|2876.75|3221.96|82.46|0.00|1030.82|1113.28|-1845.93| +2451452|39338|2786|87672|1260894|5316|32985|4|146|53|70|76.55|114.05|112.90|0.00|7903.00|5358.50|7983.50|316.12|0.00|7903.00|8219.12|2544.50| +2452604|50471|16059|48157|1761636|4715|20594|1|264|54|15|7.90|15.24|8.68|0.00|130.20|118.50|228.60|5.20|0.00|130.20|135.40|11.70| +2452604|50471|6517|48157|1761636|4715|20594|1|198|54|40|23.38|27.12|14.64|158.11|585.60|935.20|1084.80|34.19|158.11|427.49|461.68|-507.71| +2452604|50471|9159|48157|1761636|4715|20594|1|23|54|75|55.47|100.95|61.57|0.00|4617.75|4160.25|7571.25|369.42|0.00|4617.75|4987.17|457.50| +2452604|50471|1165|48157|1761636|4715|20594|1|124|54|62|69.84|138.28|113.38|0.00|7029.56|4330.08|8573.36|632.66|0.00|7029.56|7662.22|2699.48| +2452604|50471|17479|48157|1761636|4715|20594|1|210|54|34|94.14|131.79|6.58|0.00|223.72|3200.76|4480.86|0.00|0.00|223.72|223.72|-2977.04| +2452604|50471|825|48157|1761636|4715|20594|1|228|54|50|99.64|121.56|35.25|0.00|1762.50|4982.00|6078.00|123.37|0.00|1762.50|1885.87|-3219.50| +2452604|50471|702|48157|1761636|4715|20594|1|171|54|40|16.76|32.51|18.85|0.00|754.00|670.40|1300.40|52.78|0.00|754.00|806.78|83.60| +2452604|50471|3861|||4715|20594|1||54|||82.97|||3235.80|4485.00||||3235.80||| +2452604|50471|14737|48157|1761636|4715|20594|1|297|54|38|29.20|56.94|48.39|18.38|1838.82|1109.60|2163.72|18.20|18.38|1820.44|1838.64|710.84| +2452604|50471|16974||||20594|1|38|54|||85.43||0.00|588.30||||0.00||600.06|| +2451908|54460|1339|86157|1184215|7073|47178|2|203|55|16|71.32|136.93|94.48|0.00|1511.68|1141.12|2190.88|15.11|0.00|1511.68|1526.79|370.56| +2451908|54460|14923|86157|1184215|7073|47178|2|16|55|71|30.04|32.14|16.71|94.91|1186.41|2132.84|2281.94|43.66|94.91|1091.50|1135.16|-1041.34| +2451908|54460|12205|86157|1184215|7073|47178|2|85|55|90|56.92|108.14|90.83|6458.01|8174.70|5122.80|9732.60|17.16|6458.01|1716.69|1733.85|-3406.11| +2451908|54460|12170|86157|1184215|7073|47178|2|167|55|92|69.53|119.59|101.65|0.00|9351.80|6396.76|11002.28|187.03|0.00|9351.80|9538.83|2955.04| +2451908|54460|5137||1184215||47178|2|145|55|33|57.28||24.75||816.75|1890.24|2475.99||||833.08|-1073.49| +2451908|54460|3001|86157|1184215|7073|47178|2|142|55|7|57.58|91.55|54.01|0.00|378.07|403.06|640.85|11.34|0.00|378.07|389.41|-24.99| +2451908|54460|9703|86157|1184215|7073|47178|2|282|55|44|57.21|88.67|37.24|0.00|1638.56|2517.24|3901.48|81.92|0.00|1638.56|1720.48|-878.68| +2451908|54460|14135|86157|1184215|7073|47178|2|31|55|36|26.40|46.72|34.57|0.00|1244.52|950.40|1681.92|12.44|0.00|1244.52|1256.96|294.12| +2451908|54460|17677|86157|1184215|7073|47178|2|251|55|17|26.59|47.86|22.97|0.00|390.49|452.03|813.62|19.52|0.00|390.49|410.01|-61.54| +2452480|41902|8691|68383|848615|1294|3526|2|280|56|59|98.91|141.44|41.01|0.00|2419.59|5835.69|8344.96|48.39|0.00|2419.59|2467.98|-3416.10| +2452480|41902|10362|68383|848615|1294|3526|2|271|56|54|82.62|98.31|8.84|0.00|477.36|4461.48|5308.74|14.32|0.00|477.36|491.68|-3984.12| +2452480|41902|16620|68383|848615|1294|3526|2|128|56|23|9.94|17.49|13.81|0.00|317.63|228.62|402.27|28.58|0.00|317.63|346.21|89.01| +2452480|41902|8707|68383|848615|1294|3526|2|30|56|68|79.70|137.88|60.66|0.00|4124.88|5419.60|9375.84|82.49|0.00|4124.88|4207.37|-1294.72| +2452480|41902|8353|68383|848615|1294|3526|2|150|56|35|80.42|81.22|4.06|38.36|142.10|2814.70|2842.70|4.14|38.36|103.74|107.88|-2710.96| +2452480|41902|4506|68383|848615|1294|3526|2|142|56|57|69.45|125.70|90.50|0.00|5158.50|3958.65|7164.90|103.17|0.00|5158.50|5261.67|1199.85| +2452480|41902|4242|68383|848615|1294|3526|2|52|56|2|88.74|106.48|68.14|0.00|136.28|177.48|212.96|12.26|0.00|136.28|148.54|-41.20| +2452480|41902|17331|68383|848615|1294|3526|2|78|56|74|32.12|41.43|14.91|375.13|1103.34|2376.88|3065.82|65.53|375.13|728.21|793.74|-1648.67| +2451580|63306|12596|4076|1484072|2915|42911|4|9|57|91|36.67|70.77|17.69|0.00|1609.79|3336.97|6440.07|0.00|0.00|1609.79|1609.79|-1727.18| +2451580|63306|4933|4076|1484072|2915|42911|4|278|57|22|93.86|122.95|30.73|0.00|676.06|2064.92|2704.90|47.32|0.00|676.06|723.38|-1388.86| +2451580|63306|15182|4076|1484072|2915|42911|4|291|57|86|84.85|110.30|65.07|0.00|5596.02|7297.10|9485.80|0.00|0.00|5596.02|5596.02|-1701.08| +2451580|63306|3253|4076|1484072|2915|42911|4|13|57|71|91.02|108.31|80.14|0.00|5689.94|6462.42|7690.01|398.29|0.00|5689.94|6088.23|-772.48| +2451580|63306|12721|4076|1484072|2915|42911|4|75|57|62|76.40|99.32|10.92|446.84|677.04|4736.80|6157.84|0.00|446.84|230.20|230.20|-4506.60| +2451580|63306|12740|4076|1484072|2915|42911|4|225|57|52|55.58|85.03|28.05|0.00|1458.60|2890.16|4421.56|14.58|0.00|1458.60|1473.18|-1431.56| +2451580|63306|4028|4076|1484072|2915|42911|4|52|57|88|82.23|112.65|100.25|0.00|8822.00|7236.24|9913.20|529.32|0.00|8822.00|9351.32|1585.76| +2451580|63306|17729|4076|1484072|2915|42911|4|174|57|26|97.20|183.70|77.15|0.00|2005.90|2527.20|4776.20|140.41|0.00|2005.90|2146.31|-521.30| +2451580||17993||1484072|2915||4||57|99||98.76||||5256.90|||||6183.00|| +2451654||5117|21601||600||||58|2|56.01|96.33|89.58||179.16|||7.16||179.16||| +2451654|29623|15635|21601|1515277|600|723|8|243|58|6|82.45|131.09|15.73|0.00|94.38|494.70|786.54|3.77|0.00|94.38|98.15|-400.32| +2451654|29623|3848|21601|1515277|600|723|8|217|58|6|45.41|55.85|20.10|0.00|120.60|272.46|335.10|2.41|0.00|120.60|123.01|-151.86| +2451654|29623|7649|21601|1515277|600|723|8|63|58|19|44.04|50.64|42.53|0.00|808.07|836.76|962.16|64.64|0.00|808.07|872.71|-28.69| +2451654|29623|1829|21601|1515277|600|723|8|176|58|52|19.97|38.54|28.90|0.00|1502.80|1038.44|2004.08|120.22|0.00|1502.80|1623.02|464.36| +2451654|29623|16472|21601|1515277|600|723|8|39|58|6|7.58|13.87|4.02|0.00|24.12|45.48|83.22|0.24|0.00|24.12|24.36|-21.36| +2451654|29623|9743|21601|1515277|600|723|8|103|58|62|73.14|91.42|29.25|0.00|1813.50|4534.68|5668.04|18.13|0.00|1813.50|1831.63|-2721.18| +2451654|29623|16949|21601|1515277|600|723|8|206|58|35|25.76|27.04|22.17|0.00|775.95|901.60|946.40|62.07|0.00|775.95|838.02|-125.65| +2451654|29623|10802|21601|1515277|600|723|8|299|58|14|79.64|136.98|43.83|0.00|613.62|1114.96|1917.72|49.08|0.00|613.62|662.70|-501.34| +2452309|42321|12858|32944|286640|1583|29326|10|28|59|90|61.18|105.84|104.78|0.00|9430.20|5506.20|9525.60|565.81|0.00|9430.20|9996.01|3924.00| +2452309|42321|14529|32944|286640|1583|29326|10|77|59|44|42.76|69.27|8.31|277.88|365.64|1881.44|3047.88|5.26|277.88|87.76|93.02|-1793.68| +2452309|42321|3546|32944|286640|1583|29326|10|227|59|22|64.21|96.95|67.86|0.00|1492.92|1412.62|2132.90|74.64|0.00|1492.92|1567.56|80.30| +2452309|42321|5772|32944|286640|1583|29326|10|270|59|70|56.19|103.38|4.13|0.00|289.10|3933.30|7236.60|8.67|0.00|289.10|297.77|-3644.20| +2452309|42321|8010|32944|286640|1583|29326|10|63|59|46|20.15|40.09|39.28|0.00|1806.88|926.90|1844.14|72.27|0.00|1806.88|1879.15|879.98| +2452309|42321|8335|32944|286640|1583|29326|10|36|59|36|32.82|47.91|13.41|463.44|482.76|1181.52|1724.76|1.35|463.44|19.32|20.67|-1162.20| +2452309|42321|9397|32944|286640|1583|29326|10|34|59|92|70.69|139.96|19.59|0.00|1802.28|6503.48|12876.32|144.18|0.00|1802.28|1946.46|-4701.20| +2452309|42321|11916|32944|286640|1583|29326|10|244|59|19|34.31|63.47|13.32|35.43|253.08|651.89|1205.93|17.41|35.43|217.65|235.06|-434.24| +||10651|||||10||59|100|35.78|62.25|47.31|898.89|4731.00||||898.89||3985.39|| +2452309|42321|3133|32944|286640|1583|29326|10|201|59|92|14.57|27.82|19.19|0.00|1765.48|1340.44|2559.44|35.30|0.00|1765.48|1800.78|425.04| +2452309|42321|7087|32944|286640|1583|29326|10|244|59|80|58.71|64.58|0.64|0.00|51.20|4696.80|5166.40|3.58|0.00|51.20|54.78|-4645.60| +2452309|42321|9348|32944|286640|1583|29326|10|57|59|62|83.62|99.50|11.94|0.00|740.28|5184.44|6169.00|14.80|0.00|740.28|755.08|-4444.16| +2452181|62777|555|82654|81736|5618|27547|1|14|60|62|49.17|87.52|63.88|0.00|3960.56|3048.54|5426.24|39.60|0.00|3960.56|4000.16|912.02| +2452181|62777|1117|82654|81736|5618|27547|1|220|60|78|46.63|48.02|35.05|0.00|2733.90|3637.14|3745.56|54.67|0.00|2733.90|2788.57|-903.24| +2452181|62777|97|82654|81736|5618|27547|1|88|60|50|37.49|44.23|5.74|0.00|287.00|1874.50|2211.50|5.74|0.00|287.00|292.74|-1587.50| +2452181|62777|10277|82654|81736|5618|27547|1|249|60|62|28.05|33.37|25.69|0.00|1592.78|1739.10|2068.94|95.56|0.00|1592.78|1688.34|-146.32| +2452181|62777|633|82654|81736|5618|27547|1|10|60|71|96.74|162.52|87.76|0.00|6230.96|6868.54|11538.92|560.78|0.00|6230.96|6791.74|-637.58| +2452181|62777|4235|82654|81736|5618|27547|1|275|60|28|29.67|41.53|27.82|280.42|778.96|830.76|1162.84|4.98|280.42|498.54|503.52|-332.22| +||4937|||5618|||162|60|22|27.75|47.17||0.00||610.50|1037.74||0.00||744.51|| +2452181|62777|7231|82654|81736|5618|27547|1|259|60|95|45.42|88.11|79.29|0.00|7532.55|4314.90|8370.45|301.30|0.00|7532.55|7833.85|3217.65| +2452181|62777|13975|82654|81736|5618|27547|1|68|60|14|76.01|127.69|31.92|0.00|446.88|1064.14|1787.66|31.28|0.00|446.88|478.16|-617.26| +2452181|62777|10393|82654|81736|5618|27547|1|34|60|75|98.86|168.06|20.16|1375.92|1512.00|7414.50|12604.50|12.24|1375.92|136.08|148.32|-7278.42| +2452181|62777|1573|82654|81736|5618|27547|1|258|60|90|19.02|31.00|1.55|0.00|139.50|1711.80|2790.00|4.18|0.00|139.50|143.68|-1572.30| +2452181|62777|999|82654|81736|5618|27547|1|140|60|43|22.13|43.81|29.35|0.00|1262.05|951.59|1883.83|88.34|0.00|1262.05|1350.39|310.46| +2452181|62777|4041|82654|81736|5618|27547|1|285|60|25|36.26|52.21|8.35|0.00|208.75|906.50|1305.25|12.52|0.00|208.75|221.27|-697.75| +2452181|62777|17755|82654|81736|5618|27547|1|256|60|88|52.17|88.16|85.51|0.00|7524.88|4590.96|7758.08|0.00|0.00|7524.88|7524.88|2933.92| +2452181|62777|4513|82654|81736|5618|27547|1|163|60|72|79.56|146.39|139.07|0.00|10013.04|5728.32|10540.08|500.65|0.00|10013.04|10513.69|4284.72| +2452181|62777|16887|82654|81736|5618|27547|1|219|60|25|17.92|22.75|9.32|0.00|233.00|448.00|568.75|9.32|0.00|233.00|242.32|-215.00| +2451181|71311|1106|151|1065156|561|29166|1|9|61|4|78.21|95.41|31.48|0.00|125.92|312.84|381.64|1.25|0.00|125.92|127.17|-186.92| +2451181|71311|10330|151|1065156|561|29166|1|193|61|8|46.45|87.79|75.49|0.00|603.92|371.60|702.32|48.31|0.00|603.92|652.23|232.32| +2451181|71311|8924|151|1065156|561|29166|1|175|61|11|86.51|134.09|127.38|0.00|1401.18|951.61|1474.99|56.04|0.00|1401.18|1457.22|449.57| +2451181|71311|4567|151|1065156|561|29166|1|151|61|26|29.60|50.61|19.23|24.99|499.98|769.60|1315.86|4.74|24.99|474.99|479.73|-294.61| +2451181|71311|16462|151|1065156|561|29166|1|112|61|26|24.34|37.97|21.64|0.00|562.64|632.84|987.22|45.01|0.00|562.64|607.65|-70.20| +2451181|71311|6380|151|1065156|561|29166|1|233|61|77|69.52|70.91|16.30|0.00|1255.10|5353.04|5460.07|50.20|0.00|1255.10|1305.30|-4097.94| +2451181|71311|5528|151|1065156|561|29166|1|117|61|78|13.85|24.23|9.20|0.00|717.60|1080.30|1889.94|28.70|0.00|717.60|746.30|-362.70| +2451181||3362||1065156|561||1||61|48|67.92|||||3260.16|5737.44|||1548.96|1564.44|-1711.20| +2451181|71311|7591|151|1065156|561|29166|1|212|61|78|91.99|127.86|56.25|0.00|4387.50|7175.22|9973.08|0.00|0.00|4387.50|4387.50|-2787.72| +2451181|71311|2264|151|1065156|561|29166|1|219|61|36|60.94|80.44|16.89|0.00|608.04|2193.84|2895.84|42.56|0.00|608.04|650.60|-1585.80| +2452184|50871|12199|98197|570263|5176|26889|10|239|62|5|85.54|112.91|29.35|0.00|146.75|427.70|564.55|5.87|0.00|146.75|152.62|-280.95| +2452184||12407||570263|5176|26889|10|243|62|||||1799.82|2727.00|4462.80|8523.60|64.90|1799.82|927.18||| +2452184|50871|8457|98197|570263|5176|26889|10|264|62|37|99.97|103.96|94.60|35.00|3500.20|3698.89|3846.52|207.91|35.00|3465.20|3673.11|-233.69| +2452184|50871|16151|98197|570263|5176|26889|10|293|62|93|84.72|150.80|16.58|1310.64|1541.94|7878.96|14024.40|16.19|1310.64|231.30|247.49|-7647.66| +2452184|50871|15733|98197|570263|5176|26889|10|249|62|40|28.03|45.40|40.86|310.53|1634.40|1121.20|1816.00|105.90|310.53|1323.87|1429.77|202.67| +2452184||6403|||||||62||92.02|||0.00|5801.41||7251.94|232.05|0.00||6033.46|-732.01| +2452184|50871|4093|98197|570263|5176|26889|10|115|62|94|5.53|10.78|5.39|0.00|506.66|519.82|1013.32|15.19|0.00|506.66|521.85|-13.16| +2452184|50871|12163|98197|570263|5176|26889|10|287|62|29|16.70|16.86|11.29|0.00|327.41|484.30|488.94|29.46|0.00|327.41|356.87|-156.89| +2452184|50871|15399|98197|570263|5176|26889|10|122|62|74|71.70|136.94|86.27|1915.19|6383.98|5305.80|10133.56|357.50|1915.19|4468.79|4826.29|-837.01| +2452184|50871|13559|98197|570263|5176|26889|10|242|62|80|52.56|80.94|74.46|0.00|5956.80|4204.80|6475.20|357.40|0.00|5956.80|6314.20|1752.00| +2452184|50871|10149|98197|570263|5176|26889|10|143|62|75|48.36|61.41|38.07|2455.51|2855.25|3627.00|4605.75|27.98|2455.51|399.74|427.72|-3227.26| +2452543|36708|11946|96129|245000|5071|13142|8|115|63|85|94.28|94.28|32.99|0.00|2804.15|8013.80|8013.80|140.20|0.00|2804.15|2944.35|-5209.65| +2452543|36708|16569|96129|245000|5071||||63|69||125.43||||||||666.41||| +2452543|36708|5079|96129|245000|5071|13142|8|161|63|79|36.05|71.01|34.79|0.00|2748.41|2847.95|5609.79|27.48|0.00|2748.41|2775.89|-99.54| +2452543|36708|11370|96129|245000|5071|13142|8|115|63|38|56.43|86.90|26.07|0.00|990.66|2144.34|3302.20|29.71|0.00|990.66|1020.37|-1153.68| +2452543|36708|13339|96129|245000|5071|13142|8|134|63|19|19.44|25.85|4.91|0.00|93.29|369.36|491.15|0.00|0.00|93.29|93.29|-276.07| +2452543|36708|9319|96129|245000|5071|13142|8|203|63|80|9.27|13.62|10.48|0.00|838.40|741.60|1089.60|50.30|0.00|838.40|888.70|96.80| +2452543|36708|5217|96129|245000|5071|13142|8|202|63|66|15.71|18.85|7.72|0.00|509.52|1036.86|1244.10|45.85|0.00|509.52|555.37|-527.34| +2452543|36708|4488|96129|245000|5071|13142|8|207|63|73|22.85|44.78|22.83|483.31|1666.59|1668.05|3268.94|11.83|483.31|1183.28|1195.11|-484.77| +2452543|36708|15793|96129|245000|5071|13142|8|287|63|40|75.62|129.31|69.82|0.00|2792.80|3024.80|5172.40|83.78|0.00|2792.80|2876.58|-232.00| +2452543|36708|5658|96129|245000|5071|13142|8|101|63|99|88.78|111.86|64.87|0.00|6422.13|8789.22|11074.14|577.99|0.00|6422.13|7000.12|-2367.09| +2452414|44845|14407|48017|1111710|203|7449|10|91|64|8|44.36|55.45|27.17|63.03|217.36|354.88|443.60|7.71|63.03|154.33|162.04|-200.55| +2452414|44845|15867|48017|1111710|203|7449|10|163|64|59|43.77|52.96|30.71|0.00|1811.89|2582.43|3124.64|126.83|0.00|1811.89|1938.72|-770.54| +2452414|44845|11835|48017|1111710|203|7449|10|232|64|82|30.65|54.55|15.81|0.00|1296.42|2513.30|4473.10|90.74|0.00|1296.42|1387.16|-1216.88| +||1213|48017|1111710|203|||163|64||38.46|65.38||0.00|943.92|2922.96|||0.00|||| +2452414|44845|15936|48017|1111710|203|7449|10|151|64|30|56.24|64.11|55.77|0.00|1673.10|1687.20|1923.30|117.11|0.00|1673.10|1790.21|-14.10| +||13224|48017|1111710|203||||64||||||||6670.37||||6389.50|1634.64| +2452414|44845|16921|48017|1111710|203|7449|10|203|64|19|98.61|143.97|2.87|0.00|54.53|1873.59|2735.43|0.00|0.00|54.53|54.53|-1819.06| +2452414|44845|17617|48017|1111710|203|7449|10|44|64|17|91.67|119.17|29.79|0.00|506.43|1558.39|2025.89|20.25|0.00|506.43|526.68|-1051.96| +||3963|48017||||10||64||||18.03|737.78||4137.88|7985.60|15.20|737.78|380.08||| +2452414|44845|17586|48017|1111710|203|7449|10|133|64|78|77.26|124.38|68.40|0.00|5335.20|6026.28|9701.64|0.00|0.00|5335.20|5335.20|-691.08| +2452414|44845|3090|48017|1111710|203|7449|10|49|64|87|78.06|156.12|104.60|455.01|9100.20|6791.22|13582.44|0.00|455.01|8645.19|8645.19|1853.97| +2452262|53102|15177|43973|1627323|5073|39623|4|208|65|55|89.19|139.13|125.21|0.00|6886.55|4905.45|7652.15|482.05|0.00|6886.55|7368.60|1981.10| +2452262|53102|13897|43973|1627323|5073|39623|4|227|65|40|52.84|66.57|51.25|0.00|2050.00|2113.60|2662.80|82.00|0.00|2050.00|2132.00|-63.60| +2452262|53102|14905|43973|1627323|5073|39623|4|277|65|85|81.24|157.60|31.52|0.00|2679.20|6905.40|13396.00|187.54|0.00|2679.20|2866.74|-4226.20| +2452262|53102|4227|43973|1627323|5073|39623|4|264|65|42|45.38|53.54|10.17|0.00|427.14|1905.96|2248.68|12.81|0.00|427.14|439.95|-1478.82| +2452262|53102|9625|43973|1627323|5073|39623|4|69|65|51|40.95|40.95|30.30|0.00|1545.30|2088.45|2088.45|30.90|0.00|1545.30|1576.20|-543.15| +2452262|53102|17025|43973|1627323|5073|39623|4|253|65|67|25.52|37.76|32.09|107.50|2150.03|1709.84|2529.92|122.55|107.50|2042.53|2165.08|332.69| +2452262|53102|2287|43973|1627323|5073|39623|4|114|65|100|91.80|125.76|125.76|0.00|12576.00|9180.00|12576.00|251.52|0.00|12576.00|12827.52|3396.00| +2452262|53102|9847|43973|1627323|5073|39623|4|189|65|54|64.26|87.39|14.85|0.00|801.90|3470.04|4719.06|16.03|0.00|801.90|817.93|-2668.14| +2452187|39658|8197|66766|1444812|3658|43304|2|177|66|41|84.22|124.64|54.84|0.00|2248.44|3453.02|5110.24|0.00|0.00|2248.44|2248.44|-1204.58| +2452187|39658|6507|66766|1444812|3658|43304|2|293|66|76|43.81|56.07|42.05|0.00|3195.80|3329.56|4261.32|127.83|0.00|3195.80|3323.63|-133.76| +|39658|9237|66766|1444812|3658|43304|||66|29|76.94|||629.21|998.76|||14.78|629.21|||| +2452187|39658|7227|66766|1444812|3658|43304|2|224|66|66|92.15|97.67|95.71|0.00|6316.86|6081.90|6446.22|126.33|0.00|6316.86|6443.19|234.96| +2452187|39658|15629|66766|1444812|3658|43304|2|150|66|20|22.04|41.21|15.65|0.00|313.00|440.80|824.20|18.78|0.00|313.00|331.78|-127.80| +2452187|39658|11419|66766|1444812|3658|43304|2|67|66|66|10.12|12.75|12.36|350.77|815.76|667.92|841.50|4.64|350.77|464.99|469.63|-202.93| +2452187|39658|10019|66766|1444812|3658|43304|2|122|66|10|48.77|69.25|58.17|0.00|581.70|487.70|692.50|0.00|0.00|581.70|581.70|94.00| +2452187|39658|7033|66766|1444812|3658|43304|2|97|66|65|4.08|5.30|3.28|0.00|213.20|265.20|344.50|14.92|0.00|213.20|228.12|-52.00| +2452187|39658|16795|66766|1444812|3658|43304|2|153|66|28|42.00|81.06|9.72|0.00|272.16|1176.00|2269.68|0.00|0.00|272.16|272.16|-903.84| +2452187|39658|16745|66766|1444812|3658|43304|2|45|66|91|9.53|16.39|8.19|0.00|745.29|867.23|1491.49|67.07|0.00|745.29|812.36|-121.94| +2452424|57161|757|94624|1655162|6900|14591|8|228|67|77|94.12|118.59|106.73|0.00|8218.21|7247.24|9131.43|657.45|0.00|8218.21|8875.66|970.97| +2452424|57161|8871|94624|1655162|6900|14591|8|9|67|71|78.59|87.23|79.37|3606.57|5635.27|5579.89|6193.33|81.14|3606.57|2028.70|2109.84|-3551.19| +2452424|57161|3439|94624|1655162|6900|14591|8|137|67|73|23.52|35.04|23.12|0.00|1687.76|1716.96|2557.92|135.02|0.00|1687.76|1822.78|-29.20| +2452424|57161|7020|94624|1655162|6900|14591|8|182|67|79|97.01|154.24|131.10|0.00|10356.90|7663.79|12184.96|207.13|0.00|10356.90|10564.03|2693.11| +2452424|57161|5113||||14591|8||67||49.59|58.51|54.41|0.00|||702.12||0.00||659.44|| +2452424|57161|4155|94624|1655162|6900|14591|8|32|67|87|87.74|143.89|15.82|0.00|1376.34|7633.38|12518.43|0.00|0.00|1376.34|1376.34|-6257.04| +2452424|57161|15699|94624|1655162|6900|14591|8|252|67|44|34.59|41.50|35.69|0.00|1570.36|1521.96|1826.00|125.62|0.00|1570.36|1695.98|48.40| +2452424|57161|12087|94624|1655162|6900|14591|8|249|67|70|80.71|129.94|51.97|0.00|3637.90|5649.70|9095.80|145.51|0.00|3637.90|3783.41|-2011.80| +2452424|57161|7507|94624|1655162|6900|14591|8|4|67|77|26.78|53.56|1.07|0.00|82.39|2062.06|4124.12|1.64|0.00|82.39|84.03|-1979.67| +2452424|57161|8469|94624|1655162|6900|14591|||67|59||85.43|76.03|0.00|||5040.37||0.00|||637.79| +2452424|57161|2133|94624|1655162|6900|14591|8|288|67|74|71.99|76.30|14.49|0.00|1072.26|5327.26|5646.20|42.89|0.00|1072.26|1115.15|-4255.00| +2451095||7765|2564|291068|5172||||68||69.58|95.32||0.00||2087.40||70.34|0.00||1242.74|-915.00| +2451095|38136|1603|2564|291068|5172|12476|10|111|68|57|21.03|26.07|22.42|945.67|1277.94|1198.71|1485.99|3.32|945.67|332.27|335.59|-866.44| +2451095|38136|5162|2564|291068|5172|12476|10|203|68|23|83.48|155.27|10.86|0.00|249.78|1920.04|3571.21|2.49|0.00|249.78|252.27|-1670.26| +2451095|38136|2296|2564|291068|5172|12476|10|101|68|52|98.90|168.13|166.44|0.00|8654.88|5142.80|8742.76|605.84|0.00|8654.88|9260.72|3512.08| +2451095|38136|17426|2564|291068|5172|12476|10|267|68|2|48.36|59.48|1.18|0.00|2.36|96.72|118.96|0.16|0.00|2.36|2.52|-94.36| +2451095|38136|17600|2564|291068|5172|12476|10|277|68|13|52.66|57.92|10.42|101.59|135.46|684.58|752.96|1.35|101.59|33.87|35.22|-650.71| +2451095|38136|12724|2564|291068|5172|12476|10|264|68|88|9.63|11.65|8.50|0.00|748.00|847.44|1025.20|22.44|0.00|748.00|770.44|-99.44| +2451095|38136|2564|2564|291068|5172|12476|10|153|68|83|77.32|134.53|6.72|0.00|557.76|6417.56|11165.99|11.15|0.00|557.76|568.91|-5859.80| +2451095|38136|6763|2564|291068|5172|12476|10|282|68|77|96.29|112.65|68.71|0.00|5290.67|7414.33|8674.05|317.44|0.00|5290.67|5608.11|-2123.66| +2451095|38136|2035|2564|291068|5172|12476|10|173|68|22|54.35|100.54|18.09|0.00|397.98|1195.70|2211.88|23.87|0.00|397.98|421.85|-797.72| +2451095|38136|1387|2564|291068|5172|12476|10|122|68|74|90.20|103.73|94.39|0.00|6984.86|6674.80|7676.02|279.39|0.00|6984.86|7264.25|310.06| +2451095|38136|1820|2564|291068|5172|12476|10|5|68|54|55.82|98.24|33.40|0.00|1803.60|3014.28|5304.96|72.14|0.00|1803.60|1875.74|-1210.68| +2451095|38136|12526|2564|291068|5172|12476|10|86|68|3|13.06|20.11|19.30|53.26|57.90|39.18|60.33|0.23|53.26|4.64|4.87|-34.54| +2451524|50604|4267|22064|294863|2062|27261|1|244|69|10|72.37|72.37|46.31|0.00|463.10|723.70|723.70|41.67|0.00|463.10|504.77|-260.60| +2451524|50604|7822|22064|294863|2062|27261|1|8|69|30|78.10|134.33|79.25|1664.25|2377.50|2343.00|4029.90|64.19|1664.25|713.25|777.44|-1629.75| +2451524|50604|322|||||||69|45||84.53||||||7.22||||| +2451524|50604|11137|22064|294863|2062|27261|1|238|69|68|22.04|22.26|8.68|0.00|590.24|1498.72|1513.68|47.21|0.00|590.24|637.45|-908.48| +|50604|13792|||||1||69||85.64||2.89|||2483.56|||||84.64|| +2451524|50604|13489|22064|294863|2062|27261|1|264|69|66|2.68|3.16|1.92|0.00|126.72|176.88|208.56|6.33|0.00|126.72|133.05|-50.16| +2451524|50604|15448|22064|294863|2062|27261|1|149|69|16|94.38|111.36|60.13|913.97|962.08|1510.08|1781.76|2.40|913.97|48.11|50.51|-1461.97| +2451524|50604|6136|22064|294863|2062|27261|1|107|69|7|49.79|75.68|33.29|0.00|233.03|348.53|529.76|2.33|0.00|233.03|235.36|-115.50| +2451524|50604|337|22064|294863|2062|27261|1|279|69|34|20.99|27.07|12.45|0.00|423.30|713.66|920.38|12.69|0.00|423.30|435.99|-290.36| +2451524|50604|8599|22064|294863|2062|27261|1|106|69|53|56.42|58.11|26.73|0.00|1416.69|2990.26|3079.83|56.66|0.00|1416.69|1473.35|-1573.57| +2451524|50604|7264|22064|294863|2062|27261|1|34|69|67|78.29|90.03|71.12|0.00|4765.04|5245.43|6032.01|190.60|0.00|4765.04|4955.64|-480.39| +2451524|50604|4208|22064|294863|2062|27261|1|185|69|9|99.77|120.72|25.35|0.00|228.15|897.93|1086.48|15.97|0.00|228.15|244.12|-669.78| +2451880|64866|6185|4344|1865354|4317|18927|10|144|70|82|84.60|141.28|62.16|0.00|5097.12|6937.20|11584.96|101.94|0.00|5097.12|5199.06|-1840.08| +2451880|64866|2462|4344||||||70||92.70||46.16|0.00||||108.93|0.00|2723.44||-2745.86| +|64866|6425|4344|1865354|||10||70||||24.72||2076.48|3289.44|5065.20|||2076.48||| +2451880|64866|8893|4344|1865354|4317|18927|10|244|70|17|63.51|104.15|99.98|0.00|1699.66|1079.67|1770.55|152.96|0.00|1699.66|1852.62|619.99| +2451880|64866|1592|4344|1865354|4317|18927|10|262|70|53|99.59|127.47|119.82|0.00|6350.46|5278.27|6755.91|63.50|0.00|6350.46|6413.96|1072.19| +2451880|64866|3947|4344|1865354||18927||295|70||63.80||||4224.08|||126.72|||4350.80|1799.68| +2451880|64866|14549|4344|1865354|4317|18927|10|79|70|4|35.39|40.69|1.22|0.00|4.88|141.56|162.76|0.34|0.00|4.88|5.22|-136.68| +2451880|64866|17815|4344|1865354|4317|18927|10|292|70|95|36.89|42.79|4.27|0.00|405.65|3504.55|4065.05|16.22|0.00|405.65|421.87|-3098.90| +2451880|64866|9857|4344|1865354|4317|18927|10|193|70|20|54.96|97.82|41.08|0.00|821.60|1099.20|1956.40|41.08|0.00|821.60|862.68|-277.60| +2451880|64866|3296|4344|1865354|4317|18927|10|125|70|48|10.23|17.59|1.93|0.00|92.64|491.04|844.32|7.41|0.00|92.64|100.05|-398.40| +2451161|66803|2098|85239|571432|5978|30571|1|188|71|51|57.87|64.23|26.97|0.00|1375.47|2951.37|3275.73|27.50|0.00|1375.47|1402.97|-1575.90| +2451161|66803|9274|85239|571432|5978|30571|1|75|71|36|12.39|24.53|17.66|464.10|635.76|446.04|883.08|12.01|464.10|171.66|183.67|-274.38| +2451161|66803|10288|85239|571432|5978|30571|1|101|71|30|29.57|40.21|30.96|0.00|928.80|887.10|1206.30|18.57|0.00|928.80|947.37|41.70| +2451161||5858|85239||||1||71||||||||87.99|0.20||20.22||-34.11| +2451161|66803|10108|85239|571432|5978|30571|1|42|71|66|9.49|18.79|5.82|0.00|384.12|626.34|1240.14|3.84|0.00|384.12|387.96|-242.22| +2451161|66803|9997|85239|571432|5978|30571|1|210|71|26|53.28|54.87|41.15|0.00|1069.90|1385.28|1426.62|10.69|0.00|1069.90|1080.59|-315.38| +2451161|66803|17246|85239|571432|5978|30571|1|59|71|90|50.57|77.87|46.72|0.00|4204.80|4551.30|7008.30|378.43|0.00|4204.80|4583.23|-346.50| +2451161|66803|9790|85239|571432|5978|30571|1|108|71|96|37.78|68.38|7.52|0.00|721.92|3626.88|6564.48|64.97|0.00|721.92|786.89|-2904.96| +2451161|66803|4144|85239|571432|5978|30571|1|226|71|49|26.75|53.23|23.42|0.00|1147.58|1310.75|2608.27|91.80|0.00|1147.58|1239.38|-163.17| +2451161|66803|8887|85239|571432|5978|30571|1|219|71|10|61.40|91.48|32.93|0.00|329.30|614.00|914.80|3.29|0.00|329.30|332.59|-284.70| +2451161|66803|6008|85239|571432|5978|30571|1|45|71|54|38.98|66.65|63.98|0.00|3454.92|2104.92|3599.10|310.94|0.00|3454.92|3765.86|1350.00| +2451161|66803|457|85239|571432|5978|30571|1|179|71|75|27.02|42.42|41.99|0.00|3149.25|2026.50|3181.50|94.47|0.00|3149.25|3243.72|1122.75| +2451161|66803|11168|85239|571432|5978|30571|1|228|71|79|24.66|29.83|14.61|0.00|1154.19|1948.14|2356.57|80.79|0.00|1154.19|1234.98|-793.95| +||9769|||5978||1||71|79|52.15||50.35|0.00|3977.65||5849.95|238.65|0.00|3977.65||-142.20| +2451161|66803|7504|85239|571432|5978|30571|1|9|71|3|78.44|147.46|76.67|0.00|230.01|235.32|442.38|13.80|0.00|230.01|243.81|-5.31| +2451161|66803|1888|85239|571432|5978|30571|1|36|71|4|40.47|59.08|2.36|0.00|9.44|161.88|236.32|0.84|0.00|9.44|10.28|-152.44| +2451791|44195|13423|60602|1460628|5454|16633|8|215|72|62|34.39|51.58|37.13|207.18|2302.06|2132.18|3197.96|104.74|207.18|2094.88|2199.62|-37.30| +2451791|44195|10427|60602|1460628|5454|16633|8|43|72|66|55.31|101.77|15.26|0.00|1007.16|3650.46|6716.82|60.42|0.00|1007.16|1067.58|-2643.30| +2451791|44195|6113|60602|1460628|5454|16633|8|176|72|45|59.65|117.51|37.60|0.00|1692.00|2684.25|5287.95|135.36|0.00|1692.00|1827.36|-992.25| +2451791|44195|9860|60602|1460628|5454|16633|8|60|72|26|69.92|130.05|124.84|0.00|3245.84|1817.92|3381.30|129.83|0.00|3245.84|3375.67|1427.92| +2451791|44195|10753|60602|1460628|5454|16633|8|92|72|16|32.01|52.81|29.57|0.00|473.12|512.16|844.96|37.84|0.00|473.12|510.96|-39.04| +2451791|44195|12788|60602|1460628|5454|16633|8|224|72|29|34.57|40.79|19.57|0.00|567.53|1002.53|1182.91|22.70|0.00|567.53|590.23|-435.00| +2451791||5917|||5454||||72|85||8.51|4.59|0.00||436.05|||0.00|390.15||| +2451791|44195|13904|60602|1460628|5454|16633|8|77|72|37|40.39|43.62|41.87|1549.19|1549.19|1494.43|1613.94|0.00|1549.19|0.00|0.00|-1494.43| +2451791|44195|15587|60602|1460628|5454|16633|8|284|72|87|19.04|20.75|17.43|0.00|1516.41|1656.48|1805.25|75.82|0.00|1516.41|1592.23|-140.07| +2451791|44195|11741|60602|1460628|5454|16633|8|181|72|62|79.25|137.89|95.14|0.00|5898.68|4913.50|8549.18|235.94|0.00|5898.68|6134.62|985.18| +2451791|44195|1535|||5454|16633|8||72|9|69.06|||0.00|45.18|621.54||2.71|0.00||47.89|| +2451791|44195|16765|60602|1460628|5454|16633|8|256|72|56|89.44|104.64|52.32|0.00|2929.92|5008.64|5859.84|205.09|0.00|2929.92|3135.01|-2078.72| +2451791|44195|12901|60602|1460628|5454|16633|8|50|72|57|1.64|2.91|1.36|0.00|77.52|93.48|165.87|2.32|0.00|77.52|79.84|-15.96| +2451791|44195|6671|60602|1460628|5454|16633|8|195|72|13|42.82|67.65|55.47|0.00|721.11|556.66|879.45|7.21|0.00|721.11|728.32|164.45| +2451791|44195|13085|60602|1460628|5454|16633|8|74|72|94|85.13|143.01|118.69|0.00|11156.86|8002.22|13442.94|1004.11|0.00|11156.86|12160.97|3154.64| +2451434|64285|16519|5309|868000|5091|31429|2|91|73|97|82.11|86.21|65.51|0.00|6354.47|7964.67|8362.37|444.81|0.00|6354.47|6799.28|-1610.20| +2451434|64285|6862|5309|868000|5091|31429|2|199|73|58|86.48|142.69|97.02|0.00|5627.16|5015.84|8276.02|225.08|0.00|5627.16|5852.24|611.32| +|64285|17269||868000|5091||2||73|40||11.30|9.49|0.00||||0.00|0.00|379.60||| +2451434|64285|15109|5309|868000|5091|31429|2|270|73|38|53.90|76.53|74.99|2707.13|2849.62|2048.20|2908.14|1.42|2707.13|142.49|143.91|-1905.71| +2451434|64285|247|5309|868000|5091|31429|2|240|73|53|5.22|5.58|3.57|0.00|189.21|276.66|295.74|17.02|0.00|189.21|206.23|-87.45| +2451434|64285|16585|5309|868000|5091|31429|2|276|73|38|69.27|99.74|38.89|0.00|1477.82|2632.26|3790.12|118.22|0.00|1477.82|1596.04|-1154.44| +2451434|64285|10051|5309|868000|5091|31429|2|150|73|49|97.28|107.98|104.74|2771.42|5132.26|4766.72|5291.02|94.43|2771.42|2360.84|2455.27|-2405.88| +2451434|64285|3205|5309|868000|5091|31429|2|22|73|82|44.64|86.60|26.84|0.00|2200.88|3660.48|7101.20|66.02|0.00|2200.88|2266.90|-1459.60| +2451434|64285|12502|5309|868000|5091|31429|2|176|73|75|21.63|42.39|28.40|0.00|2130.00|1622.25|3179.25|149.10|0.00|2130.00|2279.10|507.75| +2451434|64285|1063|5309|868000|5091|31429|2|244|73|37|34.93|46.80|37.90|0.00|1402.30|1292.41|1731.60|42.06|0.00|1402.30|1444.36|109.89| +2451434|64285|4946|5309|868000|5091|31429|2|18|73|54|71.08|85.29|36.67|0.00|1980.18|3838.32|4605.66|99.00|0.00|1980.18|2079.18|-1858.14| +2451111|36520|3104|62205|1003527|4049|47771|1|143|74|78|52.23|104.46|82.52|0.00|6436.56|4073.94|8147.88|386.19|0.00|6436.56|6822.75|2362.62| +2451111|36520|16694|62205|1003527|4049|47771|1|122|74|72|36.04|58.38|11.09|0.00|798.48|2594.88|4203.36|39.92|0.00|798.48|838.40|-1796.40| +2451111|36520|326|62205|1003527|4049|47771|1|99|74|29|76.73|108.95|46.84|624.84|1358.36|2225.17|3159.55|7.33|624.84|733.52|740.85|-1491.65| +2451111|36520|5492|62205|1003527|4049|47771|1|80|74|54|31.24|32.80|14.10|0.00|761.40|1686.96|1771.20|45.68|0.00|761.40|807.08|-925.56| +2451111|36520|9620|62205|1003527|4049|47771|1|210|74|95|56.35|91.28|83.97|0.00|7977.15|5353.25|8671.60|558.40|0.00|7977.15|8535.55|2623.90| +2451111|36520|8653|62205|1003527|4049|47771|1|292|74|12|4.33|7.66|5.28|0.00|63.36|51.96|91.92|0.00|0.00|63.36|63.36|11.40| +2451111|36520|13208|62205|1003527|4049|47771|1|291|74|87|82.61|100.78|86.67|0.00|7540.29|7187.07|8767.86|75.40|0.00|7540.29|7615.69|353.22| +2451111|36520|3175|62205|1003527|4049|47771|1|274|74|23|50.69|82.62|9.08|0.00|208.84|1165.87|1900.26|0.00|0.00|208.84|208.84|-957.03| +2451111|36520|10069|62205|1003527|4049|47771|1|164|74|99|99.98|104.97|98.67|0.00|9768.33|9898.02|10392.03|683.78|0.00|9768.33|10452.11|-129.69| +2451111|36520|3754|62205|1003527|4049|47771|1|124|74|26|89.35|162.61|115.45|0.00|3001.70|2323.10|4227.86|180.10|0.00|3001.70|3181.80|678.60| +2451111|36520|3278|||||1||74||||56.91||||||||3872.15|-726.39| +2451111|36520|3542|62205|1003527|4049|47771|1|180|74|96|93.18|131.38|42.04|0.00|4035.84|8945.28|12612.48|242.15|0.00|4035.84|4277.99|-4909.44| +2451111|36520|7694|62205|1003527|4049|47771|1|37|74|17|36.61|69.19|67.11|0.00|1140.87|622.37|1176.23|11.40|0.00|1140.87|1152.27|518.50| +2451883|30123|14186|18184|770971|111|14098|2|135|75|63|82.77|122.49|11.02|0.00|694.26|5214.51|7716.87|20.82|0.00|694.26|715.08|-4520.25| +2451883|30123|8726|18184|770971|111|14098|2|269|75|6|25.87|26.64|13.05|0.00|78.30|155.22|159.84|0.00|0.00|78.30|78.30|-76.92| +2451883|30123|607|18184|770971|111|14098|2|67|75|20|88.61|100.12|61.07|0.00|1221.40|1772.20|2002.40|85.49|0.00|1221.40|1306.89|-550.80| +2451883|30123|4625|18184|770971|111|14098|2|159|75|73|76.04|116.34|79.11|0.00|5775.03|5550.92|8492.82|231.00|0.00|5775.03|6006.03|224.11| +2451883|30123|10217|18184|770971|111|14098|2|142|75|85|83.25|100.73|37.27|0.00|3167.95|7076.25|8562.05|285.11|0.00|3167.95|3453.06|-3908.30| +2451883|30123|2948|18184|770971|111|14098|2|151|75|25|7.48|11.07|9.40|0.00|235.00|187.00|276.75|2.35|0.00|235.00|237.35|48.00| +2451883|30123|6938|18184|770971|111|14098|2|224|75|89|20.73|36.48|26.99|0.00|2402.11|1844.97|3246.72|24.02|0.00|2402.11|2426.13|557.14| +2451883|30123|9905|18184|770971|111|14098|2|291|75|54|63.01|78.76|77.18|0.00|4167.72|3402.54|4253.04|208.38|0.00|4167.72|4376.10|765.18| +2451883|30123|6953|18184|770971|111|14098|2|58|75|71|33.34|57.67|4.03|0.00|286.13|2367.14|4094.57|2.86|0.00|286.13|288.99|-2081.01| +2451883|30123|11039|18184|770971|111|14098|2|79|75|70|87.84|110.67|13.28|0.00|929.60|6148.80|7746.90|74.36|0.00|929.60|1003.96|-5219.20| +2451883|30123|16796|18184|770971|111|14098|2|178|75|93|5.19|6.02|0.66|0.00|61.38|482.67|559.86|1.22|0.00|61.38|62.60|-421.29| +2452143|39901|257|53531|580721|1958|24348|8|284|76|5|8.47|11.34|2.15|0.00|10.75|42.35|56.70|0.10|0.00|10.75|10.85|-31.60| +2452143|39901|1503|53531|580721|1958|24348|8|192|76|97|30.22|32.93|20.41|1207.65|1979.77|2931.34|3194.21|38.60|1207.65|772.12|810.72|-2159.22| +2452143||2869||580721|||||76|32|||||||1564.80|||1267.20|1355.90|| +2452143||465||||||278|76|2|95.45||76.51|0.00|153.02||318.80||0.00|153.02||-37.88| +2452143|39901|4237|53531|580721|1958|24348|8|18|76|48|60.58|81.17|55.19|0.00|2649.12|2907.84|3896.16|158.94|0.00|2649.12|2808.06|-258.72| +|39901|1107|||1958||8|203|76|16||84.68|||||1354.88|4.24||212.12||-715.88| +2452143|39901|10331|53531|580721|1958|24348|8|157|76|3|29.09|57.01|37.05|0.00|111.15|87.27|171.03|6.66|0.00|111.15|117.81|23.88| +2452143|39901|8925|53531|580721|1958|24348|8|31|76|6|80.04|108.05|22.69|0.00|136.14|480.24|648.30|12.25|0.00|136.14|148.39|-344.10| +2452143|39901|4567|53531|580721|1958|24348|8|111|76|40|2.19|3.00|1.29|0.00|51.60|87.60|120.00|1.54|0.00|51.60|53.14|-36.00| +2452143|39901|16463|53531|580721|1958|24348|8|151|76|53|86.06|104.13|27.07|0.00|1434.71|4561.18|5518.89|100.42|0.00|1434.71|1535.13|-3126.47| +2452143|39901|6381|53531|580721|1958|24348|8|156|76|50|34.93|63.92|32.59|0.00|1629.50|1746.50|3196.00|32.59|0.00|1629.50|1662.09|-117.00| +2452143|39901|5529|53531|580721|1958|24348|8|59|76|78|49.64|65.52|53.07|0.00|4139.46|3871.92|5110.56|165.57|0.00|4139.46|4305.03|267.54| +2452143|39901|3363|53531|580721|1958|24348|8|153|76|25|89.90|106.08|77.43|1529.24|1935.75|2247.50|2652.00|20.32|1529.24|406.51|426.83|-1840.99| +2452143|39901|7591|53531|580721|1958|24348|8|207|76|27|61.86|89.69|80.72|0.00|2179.44|1670.22|2421.63|21.79|0.00|2179.44|2201.23|509.22| +2452143|39901|2265|53531|580721|1958|24348|8|18|76|98|89.70|95.08|29.47|0.00|2888.06|8790.60|9317.84|0.00|0.00|2888.06|2888.06|-5902.54| +2451083|53379|13456|19214|1611001|1988|16175|2|171|77|48|85.61|113.86|108.16|0.00|5191.68|4109.28|5465.28|51.91|0.00|5191.68|5243.59|1082.40| +2451083|53379|7448|19214|1611001|1988|16175|2|74|77|95|29.07|54.07|17.84|0.00|1694.80|2761.65|5136.65|84.74|0.00|1694.80|1779.54|-1066.85| +2451083|53379|2713|19214|1611001|1988|16175|2|284|77|85|85.72|129.43|56.94|0.00|4839.90|7286.20|11001.55|387.19|0.00|4839.90|5227.09|-2446.30| +2451083|53379|1399|19214|1611001|1988|16175|2|161|77|34|96.21|182.79|29.24|596.49|994.16|3271.14|6214.86|3.97|596.49|397.67|401.64|-2873.47| +2451083|53379|7339|19214|1611001|1988|16175|2|54|77|88|31.13|59.14|13.60|155.58|1196.80|2739.44|5204.32|10.41|155.58|1041.22|1051.63|-1698.22| +2451083|53379|8686|19214|1611001|1988|16175|2|225|77|38|45.30|84.25|37.91|0.00|1440.58|1721.40|3201.50|72.02|0.00|1440.58|1512.60|-280.82| +2451083|53379|12439|19214|1611001|1988|16175|2|9|77|95|74.32|115.93|31.30|0.00|2973.50|7060.40|11013.35|118.94|0.00|2973.50|3092.44|-4086.90| +2451083|53379|9220|19214|1611001|1988|16175|2|270|77|90|87.41|138.10|75.95|0.00|6835.50|7866.90|12429.00|136.71|0.00|6835.50|6972.21|-1031.40| +2451083|53379|992|19214|1611001|1988|16175|2|295|77|62|21.65|33.34|3.33|0.00|206.46|1342.30|2067.08|10.32|0.00|206.46|216.78|-1135.84| +2451083||7486|19214|1611001|1988||2||77|49||84.08|||535.57|||26.77||535.57|562.34|| +2451083|53379|6289|19214|1611001|1988|16175|2|217|77|30|26.16|32.70|30.41|0.00|912.30|784.80|981.00|54.73|0.00|912.30|967.03|127.50| +2451083|53379|3868|19214|1611001|1988|16175|2|140|77|89|3.72|6.28|1.31|0.00|116.59|331.08|558.92|9.32|0.00|116.59|125.91|-214.49| +2451083|53379|16687|19214|1611001|1988|16175|2|175|77|16|67.63|119.02|86.88|0.00|1390.08|1082.08|1904.32|97.30|0.00|1390.08|1487.38|308.00| +2451083|53379|11918|19214|1611001|1988|16175|2|167|77|36|25.95|42.03|19.75|0.00|711.00|934.20|1513.08|28.44|0.00|711.00|739.44|-223.20| +2451083|53379|14815|19214|1611001|1988|16175|2|277|77|18|69.28|87.29|52.37|0.00|942.66|1247.04|1571.22|84.83|0.00|942.66|1027.49|-304.38| +2451063|30397|6110|10282|743469|2609|19796|4|249|78|43|28.45|40.68|10.98|0.00|472.14|1223.35|1749.24|23.60|0.00|472.14|495.74|-751.21| +2451063|30397|6044|10282|743469|2609|19796|4|213|78|59|15.94|29.01|25.52|240.90|1505.68|940.46|1711.59|50.59|240.90|1264.78|1315.37|324.32| +2451063|30397|11209|10282|743469|2609|19796|4|187|78|7|88.17|166.64|121.64|664.15|851.48|617.19|1166.48|1.87|664.15|187.33|189.20|-429.86| +2451063|30397|6500|10282|743469|2609|19796|4|289|78|76|38.42|68.77|63.26|4375.06|4807.76|2919.92|5226.52|12.98|4375.06|432.70|445.68|-2487.22| +2451063|30397|3304|10282|743469|2609|19796|4|252|78|50|61.49|75.01|72.00|0.00|3600.00|3074.50|3750.50|216.00|0.00|3600.00|3816.00|525.50| +2451063|30397|7576|10282|743469|2609|19796|4|180|78|87|60.62|83.65|15.89|0.00|1382.43|5273.94|7277.55|41.47|0.00|1382.43|1423.90|-3891.51| +2451063|30397|14996|10282|743469|2609|19796|4|213|78|39|36.47|53.24|8.51|0.00|331.89|1422.33|2076.36|3.31|0.00|331.89|335.20|-1090.44| +2451063|30397|901|10282|743469|2609|19796|4|49|78|3|54.50|105.73|88.81|223.80|266.43|163.50|317.19|1.70|223.80|42.63|44.33|-120.87| +2451063|30397|12706|10282|743469|2609|19796|4|124|78|19|81.10|116.78|82.91|0.00|1575.29|1540.90|2218.82|94.51|0.00|1575.29|1669.80|34.39| +2451063|30397|5965|10282|743469|2609|19796|4|8|78|78|7.47|13.22|0.26|0.00|20.28|582.66|1031.16|1.62|0.00|20.28|21.90|-562.38| +2451063|30397|8611|10282|743469|2609|19796|4|40|78|79|95.42|163.16|76.68|0.00|6057.72|7538.18|12889.64|0.00|0.00|6057.72|6057.72|-1480.46| +2451063|30397|3856|10282|743469|2609|19796|4|9|78|27|60.10|87.14|73.19|0.00|1976.13|1622.70|2352.78|158.09|0.00|1976.13|2134.22|353.43| +2451063|30397|10507|10282|743469|2609|19796|4|56|78|6|50.83|57.94|36.50|0.00|219.00|304.98|347.64|6.57|0.00|219.00|225.57|-85.98| +2452280|32066|16519|62017|1062416|1215|43261|7|117|79|9|69.84|98.47|12.80|0.00|115.20|628.56|886.23|4.60|0.00|115.20|119.80|-513.36| +2452280|32066|6864|62017|1062416|1215|43261|7|1|79|1|86.12|105.92|68.84|0.00|68.84|86.12|105.92|1.37|0.00|68.84|70.21|-17.28| +2452280|32066|17269|62017|1062416|1215|43261|7|216|79|81|5.88|9.52|1.61|0.00|130.41|476.28|771.12|11.73|0.00|130.41|142.14|-345.87| +2452280|32066|15109|||1215|43261|7|150|79||45.07|52.73|20.03||620.93|1397.17||||||| +2452280|32066|247|62017||||7||79|||113.96|28.49||227.92|660.64|911.68|||227.92||-432.72| +2452280|32066|16585|62017|1062416|1215|43261|7|252|79|93|70.70|114.53|9.16|0.00|851.88|6575.10|10651.29|0.00|0.00|851.88|851.88|-5723.22| +2452280|32066|10051|62017|1062416|1215|43261|7|156|79|10|94.53|124.77|19.96|0.00|199.60|945.30|1247.70|17.96|0.00|199.60|217.56|-745.70| +2452280|32066|3205|62017|1062416|1215|43261|7|133|79|48|21.34|26.67|15.20|0.00|729.60|1024.32|1280.16|7.29|0.00|729.60|736.89|-294.72| +2452280|32066|12504|62017|1062416|1215|43261|7|141|79|81|14.87|27.80|5.28|0.00|427.68|1204.47|2251.80|12.83|0.00|427.68|440.51|-776.79| +2452280|32066|1063|62017|1062416|1215|43261|7|224|79|85|77.51|148.81|139.88|0.00|11889.80|6588.35|12648.85|118.89|0.00|11889.80|12008.69|5301.45| +2452280|32066|4947|62017|1062416|1215|43261|7|14|79|35|99.77|180.58|74.03|1114.15|2591.05|3491.95|6320.30|44.30|1114.15|1476.90|1521.20|-2015.05| +2452280|32066|10524|62017|1062416|1215|43261|7|202|79|36|21.73|22.81|5.24|0.00|188.64|782.28|821.16|11.31|0.00|188.64|199.95|-593.64| +2452280|32066|14322|62017|1062416|1215|43261|7|18|79|41|58.88|109.51|64.61|0.00|2649.01|2414.08|4489.91|185.43|0.00|2649.01|2834.44|234.93| +2452280|32066|15888|62017|1062416|1215|43261|7|154|79|58|99.75|146.63|23.46|1156.57|1360.68|5785.50|8504.54|18.36|1156.57|204.11|222.47|-5581.39| +2452280|32066|15498|62017|1062416|1215|43261|7|181|79|3|94.64|147.63|87.10|0.00|261.30|283.92|442.89|23.51|0.00|261.30|284.81|-22.62| +2451143|67039|7231|17368|633451|1887|30323|10|27|80|49|76.07|116.38|95.43|0.00|4676.07|3727.43|5702.62|327.32|0.00|4676.07|5003.39|948.64| +2451143|67039|13975|17368|633451|1887|30323|10|162|80|13|83.81|136.61|121.58|0.00|1580.54|1089.53|1775.93|47.41|0.00|1580.54|1627.95|491.01| +2451143|67039|10393|17368|633451|1887|30323|10|283|80|5|71.37|97.06|67.94|0.00|339.70|356.85|485.30|20.38|0.00|339.70|360.08|-17.15| +2451143|67039|1573|17368|633451|1887|30323|10|137|80|40|43.33|83.62|81.94|0.00|3277.60|1733.20|3344.80|196.65|0.00|3277.60|3474.25|1544.40| +2451143|67039|998|17368|633451|1887|30323|10|297|80|93|69.32|75.55|24.93|0.00|2318.49|6446.76|7026.15|208.66|0.00|2318.49|2527.15|-4128.27| +2451143|67039|4040|17368|633451|1887|30323|10|94|80|66|15.01|27.91|3.07|0.00|202.62|990.66|1842.06|14.18|0.00|202.62|216.80|-788.04| +2451143|67039|17755|17368|633451|1887|30323|10|147|80|94|56.04|64.44|12.24|0.00|1150.56|5267.76|6057.36|46.02|0.00|1150.56|1196.58|-4117.20| +2451143|67039|4513||||30323|10||80||76.02||112.82|0.00|9251.24|6233.64|9537.42||0.00||9898.82|3017.60| +2451143|67039|16886|17368|633451|1887|30323|10|189|80|77|89.22|146.32|7.31|135.08|562.87|6869.94|11266.64|4.27|135.08|427.79|432.06|-6442.15| +2451143|67039|1519|17368|633451|1887|30323|10|147|80|25|66.36|130.06|58.52|0.00|1463.00|1659.00|3251.50|0.00|0.00|1463.00|1463.00|-196.00| +2451143|67039|7610|17368|633451|1887|30323|10|201|80|37|24.62|26.09|21.13|0.00|781.81|910.94|965.33|15.63|0.00|781.81|797.44|-129.13| +||12968|17368|633451||||197|80|||28.84|16.15|||2103.93|2797.48|140.98||||| +2451143|67039|4622|17368|633451|1887|30323|10|249|80|1|60.10|110.58|33.17|0.00|33.17|60.10|110.58|2.32|0.00|33.17|35.49|-26.93| +2451143|67039|17308|17368|633451|1887|30323|10|57|80|29|94.38|159.50|71.77|0.00|2081.33|2737.02|4625.50|104.06|0.00|2081.33|2185.39|-655.69| +2451143|67039|13717|17368|633451|1887|30323|10|137|80|91|60.42|116.00|106.72|0.00|9711.52|5498.22|10556.00|97.11|0.00|9711.52|9808.63|4213.30| +2451143|67039|16363|17368|633451|1887|30323|10|262|80|84|84.80|109.39|25.15|0.00|2112.60|7123.20|9188.76|84.50|0.00|2112.60|2197.10|-5010.60| +2451167|61212|5078|24625|258943|3661|29179|7|258|81|75|33.72|35.06|11.21|0.00|840.75|2529.00|2629.50|58.85|0.00|840.75|899.60|-1688.25| +2451167||11368|24625||3661|29179|7||81|26||43.18||582.98|594.88|951.60||0.11|582.98||12.01|-939.70| +2451167|61212|13339|24625|258943|3661|29179|7|261|81|6|4.63|8.19|7.20|36.72|43.20|27.78|49.14|0.25|36.72|6.48|6.73|-21.30| +2451167|61212|9319|24625|258943|3661|29179|7|79|81|36|91.74|143.11|64.39|1831.25|2318.04|3302.64|5151.96|14.60|1831.25|486.79|501.39|-2815.85| +2451167|61212|5216|24625|258943|3661|29179|7|81|81|64|4.59|6.33|6.26|0.00|400.64|293.76|405.12|4.00|0.00|400.64|404.64|106.88| +2451167|61212|4486|24625|258943|3661|29179|7|64|81|31|63.84|79.16|3.95|0.00|122.45|1979.04|2453.96|7.34|0.00|122.45|129.79|-1856.59| +2451167|61212|15793|24625|258943|3661|29179|7|229|81|8|5.61|10.09|10.09|0.00|80.72|44.88|80.72|1.61|0.00|80.72|82.33|35.84| +2451167|61212|5656|24625|258943|3661|29179|7|136|81|24|40.61|76.34|0.00|0.00|0.00|974.64|1832.16|0.00|0.00|0.00|0.00|-974.64| +2451167|61212|8918|24625|258943|3661|29179|7|62|81|37|86.54|154.04|61.61|0.00|2279.57|3201.98|5699.48|205.16|0.00|2279.57|2484.73|-922.41| +2451167|61212|7166|24625|258943|3661|29179|7|259|81|7|22.87|40.93|0.00|0.00|0.00|160.09|286.51|0.00|0.00|0.00|0.00|-160.09| +2451167|61212|11107|24625|258943|3661|29179|7|29|81|36|47.86|86.14|74.08|0.00|2666.88|1722.96|3101.04|160.01|0.00|2666.88|2826.89|943.92| +2451167|61212|7663|24625|258943|3661|29179|7|284|81|79|52.07|59.35|16.61|0.00|1312.19|4113.53|4688.65|104.97|0.00|1312.19|1417.16|-2801.34| +2451363|75450|7862|20896|1350703|157|46411|4|36|82|75|21.65|25.98|21.30|0.00|1597.50|1623.75|1948.50|79.87|0.00|1597.50|1677.37|-26.25| +2451363|75450|14998|20896|1350703|157|46411|4|249|82|49|52.59|57.32|33.81|0.00|1656.69|2576.91|2808.68|149.10|0.00|1656.69|1805.79|-920.22| +2451363|75450|13138|20896|1350703|157|46411|4|125|82|59|31.81|41.67|11.66|564.11|687.94|1876.79|2458.53|6.19|564.11|123.83|130.02|-1752.96| +2451363|75450|17041|20896|1350703|157|46411|4|17|82|18|4.71|7.25|4.93|0.00|88.74|84.78|130.50|6.21|0.00|88.74|94.95|3.96| +2451363|75450|2572|20896|1350703|157|46411|4|154|82|53|55.41|109.71|86.67|0.00|4593.51|2936.73|5814.63|91.87|0.00|4593.51|4685.38|1656.78| diff --git a/dev-support/jenkins-common.sh b/dev-support/jenkins-common.sh new file mode 100644 index 0000000..f49f099 --- /dev/null +++ b/dev-support/jenkins-common.sh @@ -0,0 +1,92 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +fail() { + echo "$@" 1>&2 + exit 1 +} + +# Parses the JIRA/patch to find relavent information. +# Exports two variables of import: +# * BUILD_PROFILE - the profile which the ptest server understands +# * BUILD_OPTS - additional test options to be sent to ptest cli +process_jira() { + test -n "$BRANCH" || fail "BRANCH must be specified" + test -n "$JIRA_ROOT_URL" || fail "JIRA_ROOT_URL must be specified" + test -n "$JIRA_NAME" || fail "API_PASSWORD must be specified" + JIRA_TEXT=$(mktemp) + trap "rm -f $JIRA_TEXT" EXIT + curl -s -S --location --retry 3 "${JIRA_ROOT_URL}/jira/browse/${JIRA_NAME}" > $JIRA_TEXT + if [[ "${CHECK_NO_PRECOMMIT_TESTS}" == "true" ]] && grep -q "NO PRECOMMIT TESTS" $JIRA_TEXT + then + fail "Test $JIRA_NAME has tag NO PRECOMMIT TESTS" + fi + # ensure the patch is actually in the correct state + if ! grep -q 'Patch Available' $JIRA_TEXT + then + fail "$JIRA_NAME is not \"Patch Available\". Exiting." + fi + # pull attachments from JIRA (hack stolen from hadoop since rest api doesn't show attachments) + PATCH_URL=$(grep -o '"/jira/secure/attachment/[0-9]*/[^"]*' $JIRA_TEXT | \ + grep -v -e 'htm[l]*$' | sort | tail -1 | \ + grep -o '/jira/secure/attachment/[0-9]*/[^"]*') + if [[ -z "$PATCH_URL" ]] + then + fail "Unable to find attachment for $JIRA_NAME" + fi + # ensure attachment has not already been tested + ATTACHMENT_ID=$(basename $(dirname $PATCH_URL)) + if grep -q "ATTACHMENT ID: $ATTACHMENT_ID" $JIRA_TEXT + then + fail "Attachment $ATTACHMENT_ID is already tested for $JIRA_NAME" + fi + # validate the patch name, parse branch if needed + shopt -s nocasematch + PATCH_NAME=$(basename $PATCH_URL) + # Test examples: + # HIVE-123.patch HIVE-123.1.patch HIVE-123.D123.patch HIVE-123.D123.1.patch HIVE-123-tez.patch HIVE-123.1-tez.patch + # HIVE-XXXX.patch, HIVE-XXXX.XX.patch HIVE-XXXX.XX-branch.patch HIVE-XXXX-branch.patch + if [[ $PATCH_NAME =~ ^HIVE-[0-9]+(\.[0-9]+)?(-[a-z0-9-]+)?\.(patch|patch.\txt)$ ]] + then + if [[ -n "${BASH_REMATCH[2]}" ]] + then + BRANCH=${BASH_REMATCH[2]#*-} + else + echo "Assuming branch $BRANCH" + fi + # HIVE-XXXX.DXXXX.patch or HIVE-XXXX.DXXXX.XX.patch + elif [[ $PATCH_NAME =~ ^(HIVE-[0-9]+\.)?D[0-9]+(\.[0-9]+)?\.(patch|patch.\txt)$ ]] + then + echo "Assuming branch $BRANCH" + else + fail "Patch $PATCH_NAME does not appear to be a patch" + fi + shopt -u nocasematch + # append mr2 if needed + if [[ $BRANCH =~ (mr1|mr2)$ ]] + then + profile=$BRANCH + else + profile=${BRANCH}-mr2 + fi + export BUILD_PROFILE=$profile + build_opts="" + if grep -q "CLEAR LIBRARY CACHE" $JIRA_TEXT + then + echo "Clearing library cache before starting test" + build_opts="--clearLibraryCache" + fi + export BUILD_OPTS=$build_opts +} diff --git a/dev-support/jenkins-execute-build.sh b/dev-support/jenkins-execute-build.sh new file mode 100644 index 0000000..06b343a --- /dev/null +++ b/dev-support/jenkins-execute-build.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e +. jenkins-common.sh +test -n "$BRANCH" || fail "BRANCH must be specified" +test -n "$API_ENDPOINT" || fail "API_ENDPOINT must be specified" +test -n "$LOG_ENDPOINT" || fail "LOG_ENDPOINT must be specified" +test -n "$API_PASSWORD" || fail "API_PASSWORD must be specified" +export JIRA_NAME="HIVE-${ISSUE_NUM}" +export ROOT=$PWD +export JIRA_ROOT_URL="https://issues.apache.org" +export BUILD_TAG="${BUILD_TAG##jenkins-}" +echo $JIRA_NAME +set -x +env + +process_jira + +test -d hive/build/ || mkdir -p hive/build/ +cd hive/build/ +rm -rf ptest2 +svn co http://svn.apache.org/repos/asf/hive/trunk/testutils/ptest2/ ptest2 +cd ptest2 + +# sanity check the profile +case "$BUILD_PROFILE" in + trunk-mr1);; + trunk-mr2);; + *) + echo "Unknown profile '$BUILD_PROFILE'" + exit 1 + ;; +esac +mvn clean package -DskipTests -Drat.numUnapprovedLicenses=1000 -Dmaven.repo.local=$WORKSPACE/.m2 +set +e +java -cp "target/hive-ptest-1.0-classes.jar:target/lib/*" org.apache.hive.ptest.api.client.PTestClient --endpoint "$API_ENDPOINT" \ + --logsEndpoint "$LOG_ENDPOINT" \ + --command testStart \ + --profile $profile \ + --password $API_PASSWORD \ + --outputDir target/ \ + --testHandle "$BUILD_TAG" \ + --patch "${JIRA_ROOT_URL}${PATCH_URL}" \ + --jira "$JIRA_NAME" ${BUILD_OPTS} "$@" +ret=$? +cd target/ +if [[ -f test-results.tar.gz ]] +then + rm -rf $ROOT/hive/build/test-results/ + tar zxf test-results.tar.gz -C $ROOT/hive/build/ +fi +exit $ret diff --git a/dev-support/jenkins-submit-build.sh b/dev-support/jenkins-submit-build.sh new file mode 100644 index 0000000..d44ee03 --- /dev/null +++ b/dev-support/jenkins-submit-build.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e +. jenkins-common.sh +export JIRA_NAME="HIVE-${ISSUE_NUM}" +export JIRA_ROOT_URL="https://issues.apache.org" +export BRANCH=trunk +echo $JIRA_NAME + +process_jira + +# sanity check the profile +case "$BUILD_PROFILE" in + trunk-mr1|trunk-mr2) + test -n "$TRUNK_URL" || fail "TRUNK_URL must be specified" + url="$TRUNK_URL&ISSUE_NUM=$ISSUE_NUM" + curl -v -i "$url" + exit 0 + ;; + spark-mr2|spark2-mr2) + test -n "$SPARK_URL" || fail "SPARK_URL must be specified" + url="$SPARK_URL&ISSUE_NUM=$ISSUE_NUM" + curl -v -i "$url" + exit 0 + ;; + *) + echo "Unknown profile '$BUILD_PROFILE'" + exit 1 + ;; +esac diff --git a/hbase-handler/pom.xml b/hbase-handler/pom.xml index d9f0763..d55ec5a 100644 --- a/hbase-handler/pom.xml +++ b/hbase-handler/pom.xml @@ -19,7 +19,7 @@ org.apache.hive hive - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../pom.xml diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/ColumnMappings.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/ColumnMappings.java index fbd1308..5cb3752 100644 --- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/ColumnMappings.java +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/ColumnMappings.java @@ -34,7 +34,9 @@ import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import com.google.common.collect.Iterators; @@ -42,11 +44,17 @@ public class ColumnMappings implements Iterable { private final int keyIndex; + private final int timestampIndex; private final ColumnMapping[] columnsMapping; public ColumnMappings(List columnMapping, int keyIndex) { + this(columnMapping, keyIndex, -1); + } + + public ColumnMappings(List columnMapping, int keyIndex, int timestampIndex) { this.columnsMapping = columnMapping.toArray(new ColumnMapping[columnMapping.size()]); this.keyIndex = keyIndex; + this.timestampIndex = timestampIndex; } @Override @@ -109,7 +117,9 @@ void setHiveColumnDescription(String serdeName, // where key extends LazyPrimitive and thus has type Category.PRIMITIVE for (int i = 0; i < columnNames.size(); i++) { ColumnMapping colMap = columnsMapping[i]; - if (colMap.qualifierName == null && !colMap.hbaseRowKey) { + colMap.columnName = columnNames.get(i); + colMap.columnType = columnTypes.get(i); + if (colMap.qualifierName == null && !colMap.hbaseRowKey && !colMap.hbaseTimestamp) { TypeInfo typeInfo = columnTypes.get(i); if ((typeInfo.getCategory() != ObjectInspector.Category.MAP) || (((MapTypeInfo) typeInfo).getMapKeyTypeInfo().getCategory() @@ -122,8 +132,14 @@ void setHiveColumnDescription(String serdeName, + typeInfo.getTypeName()); } } - colMap.columnName = columnNames.get(i); - colMap.columnType = columnTypes.get(i); + if (colMap.hbaseTimestamp) { + TypeInfo typeInfo = columnTypes.get(i); + if (!colMap.isCategory(PrimitiveCategory.TIMESTAMP) && + !colMap.isCategory(PrimitiveCategory.LONG)) { + throw new SerDeException(serdeName + ": timestamp columns should be of " + + "timestamp or bigint type, but is mapped to " + typeInfo.getTypeName()); + } + } } } @@ -299,10 +315,18 @@ public ColumnMapping getKeyMapping() { return columnsMapping[keyIndex]; } + public ColumnMapping getTimestampMapping() { + return timestampIndex < 0 ? null : columnsMapping[timestampIndex]; + } + public int getKeyIndex() { return keyIndex; } + public int getTimestampIndex() { + return timestampIndex; + } + public ColumnMapping[] getColumnsMapping() { return columnsMapping; } @@ -326,6 +350,7 @@ public int getKeyIndex() { byte[] qualifierNameBytes; List binaryStorage; boolean hbaseRowKey; + boolean hbaseTimestamp; String mappingSpec; String qualifierPrefix; byte[] qualifierPrefixBytes; @@ -377,5 +402,14 @@ public String getQualifierPrefix() { public boolean isCategory(ObjectInspector.Category category) { return columnType.getCategory() == category; } + + public boolean isCategory(PrimitiveCategory category) { + return columnType.getCategory() == ObjectInspector.Category.PRIMITIVE && + ((PrimitiveTypeInfo)columnType).getPrimitiveCategory() == category; + } + + public boolean isComparable() { + return binaryStorage.get(0) || isCategory(PrimitiveCategory.STRING); + } } } diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/DefaultHBaseKeyFactory.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/DefaultHBaseKeyFactory.java index 12c5377..98bc73f 100644 --- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/DefaultHBaseKeyFactory.java +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/DefaultHBaseKeyFactory.java @@ -21,17 +21,18 @@ import java.io.IOException; import java.util.Properties; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.lazy.LazyFactory; import org.apache.hadoop.hive.serde2.lazy.LazyObjectBase; -import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; +import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.SerDeParameters; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; public class DefaultHBaseKeyFactory extends AbstractHBaseKeyFactory implements HBaseKeyFactory { - protected LazySimpleSerDe.SerDeParameters serdeParams; + protected SerDeParameters serdeParams; protected HBaseRowSerializer serializer; @Override @@ -56,4 +57,12 @@ public LazyObjectBase createKey(ObjectInspector inspector) throws SerDeException public byte[] serializeKey(Object object, StructField field) throws IOException { return serializer.serializeKeyField(object, field, keyMapping); } + + @VisibleForTesting + static DefaultHBaseKeyFactory forTest(SerDeParameters params, ColumnMappings mappings) { + DefaultHBaseKeyFactory factory = new DefaultHBaseKeyFactory(); + factory.serdeParams = params; + factory.keyMapping = mappings.getKeyMapping(); + return factory; + } } diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseKeyFactory.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseKeyFactory.java index 251d22e..4817b43 100644 --- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseKeyFactory.java +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseKeyFactory.java @@ -59,8 +59,7 @@ * serialize hive object in internal format of custom key * * @param object - * @param inspector - * @param output + * @param field * * @return true if it's not null * @throws java.io.IOException diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseRowSerializer.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseRowSerializer.java index c6c42b4..3bbab20 100644 --- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseRowSerializer.java +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseRowSerializer.java @@ -35,7 +35,9 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.io.Writable; public class HBaseRowSerializer { @@ -45,7 +47,9 @@ private final LazySimpleSerDe.SerDeParameters serdeParam; private final int keyIndex; + private final int timestampIndex; private final ColumnMapping keyMapping; + private final ColumnMapping timestampMapping; private final ColumnMapping[] columnMappings; private final byte[] separators; // the separators array private final boolean escaped; // whether we need to escape the data when writing out @@ -66,8 +70,10 @@ public HBaseRowSerializer(HBaseSerDeParameters hbaseParam) { this.escapeChar = serdeParam.getEscapeChar(); this.needsEscape = serdeParam.getNeedsEscape(); this.keyIndex = hbaseParam.getKeyIndex(); + this.timestampIndex = hbaseParam.getTimestampIndex(); this.columnMappings = hbaseParam.getColumnMappings().getColumnsMapping(); this.keyMapping = hbaseParam.getColumnMappings().getKeyMapping(); + this.timestampMapping = hbaseParam.getColumnMappings().getTimestampMapping(); this.putTimestamp = hbaseParam.getPutTimestamp(); } @@ -81,25 +87,36 @@ public Writable serialize(Object obj, ObjectInspector objInspector) throws Excep // Prepare the field ObjectInspectors StructObjectInspector soi = (StructObjectInspector) objInspector; List fields = soi.getAllStructFieldRefs(); - List list = soi.getStructFieldsDataAsList(obj); + List values = soi.getStructFieldsDataAsList(obj); StructField field = fields.get(keyIndex); - Object value = list.get(keyIndex); + Object value = values.get(keyIndex); byte[] key = keyFactory.serializeKey(value, field); if (key == null) { throw new SerDeException("HBase row key cannot be NULL"); } + long timestamp = putTimestamp; + if (timestamp < 0 && timestampIndex >= 0) { + ObjectInspector inspector = fields.get(timestampIndex).getFieldObjectInspector(); + value = values.get(timestampIndex); + if (inspector instanceof LongObjectInspector) { + timestamp = ((LongObjectInspector)inspector).get(value); + } else { + PrimitiveObjectInspector primitive = (PrimitiveObjectInspector) inspector; + timestamp = PrimitiveObjectInspectorUtils.getTimestamp(value, primitive).getTime(); + } + } - Put put = putTimestamp >= 0 ? new Put(key, putTimestamp) : new Put(key); + Put put = timestamp >= 0 ? new Put(key, timestamp) : new Put(key); // Serialize each field for (int i = 0; i < fields.size(); i++) { - if (i == keyIndex) { + if (i == keyIndex || i == timestampIndex) { continue; } field = fields.get(i); - value = list.get(i); + value = values.get(i); serializeField(value, field, columnMappings[i], put); } diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java index aedd843..47e20d5 100644 --- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java @@ -28,9 +28,11 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hive.hbase.ColumnMappings.ColumnMapping; import org.apache.hadoop.hive.ql.plan.TableDesc; +import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.AbstractSerDe; import org.apache.hadoop.hive.serde2.SerDe; import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.SerDeSpec; import org.apache.hadoop.hive.serde2.SerDeStats; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazySimpleStructObjectInspector; @@ -42,6 +44,27 @@ * HBaseSerDe can be used to serialize object into an HBase table and * deserialize objects from an HBase table. */ +@SerDeSpec(schemaProps = { + serdeConstants.LIST_COLUMNS, serdeConstants.LIST_COLUMN_TYPES, + serdeConstants.FIELD_DELIM, serdeConstants.COLLECTION_DELIM, serdeConstants.MAPKEY_DELIM, + serdeConstants.SERIALIZATION_FORMAT, serdeConstants.SERIALIZATION_NULL_FORMAT, + serdeConstants.SERIALIZATION_LAST_COLUMN_TAKES_REST, + serdeConstants.ESCAPE_CHAR, + serdeConstants.SERIALIZATION_ENCODING, + LazySimpleSerDe.SERIALIZATION_EXTEND_NESTING_LEVELS, + HBaseSerDe.HBASE_COLUMNS_MAPPING, + HBaseSerDe.HBASE_TABLE_NAME, + HBaseSerDe.HBASE_TABLE_DEFAULT_STORAGE_TYPE, + HBaseSerDe.HBASE_KEY_COL, + HBaseSerDe.HBASE_PUT_TIMESTAMP, + HBaseSerDe.HBASE_COMPOSITE_KEY_CLASS, + HBaseSerDe.HBASE_COMPOSITE_KEY_TYPES, + HBaseSerDe.HBASE_COMPOSITE_KEY_FACTORY, + HBaseSerDe.HBASE_STRUCT_SERIALIZER_CLASS, + HBaseSerDe.HBASE_SCAN_CACHE, + HBaseSerDe.HBASE_SCAN_CACHEBLOCKS, + HBaseSerDe.HBASE_SCAN_BATCH, + HBaseSerDe.HBASE_AUTOGENERATE_STRUCT}) public class HBaseSerDe extends AbstractSerDe { public static final Log LOG = LogFactory.getLog(HBaseSerDe.class); @@ -49,6 +72,7 @@ public static final String HBASE_TABLE_NAME = "hbase.table.name"; public static final String HBASE_TABLE_DEFAULT_STORAGE_TYPE = "hbase.table.default.storage.type"; public static final String HBASE_KEY_COL = ":key"; + public static final String HBASE_TIMESTAMP_COL = ":timestamp"; public static final String HBASE_PUT_TIMESTAMP = "hbase.put.timestamp"; public static final String HBASE_COMPOSITE_KEY_CLASS = "hbase.composite.key.class"; public static final String HBASE_COMPOSITE_KEY_TYPES = "hbase.composite.key.types"; @@ -98,8 +122,7 @@ public void initialize(Configuration conf, Properties tbl) serdeParams.getValueFactories()); cachedHBaseRow = new LazyHBaseRow( - (LazySimpleStructObjectInspector) cachedObjectInspector, - serdeParams.getKeyIndex(), serdeParams.getKeyFactory(), serdeParams.getValueFactories()); + (LazySimpleStructObjectInspector) cachedObjectInspector, serdeParams); serializer = new HBaseRowSerializer(serdeParams); @@ -135,6 +158,7 @@ public static ColumnMappings parseColumnsMapping( } int rowKeyIndex = -1; + int timestampIndex = -1; List columnsMapping = new ArrayList(); String[] columnSpecs = columnsMappingSpec.split(","); @@ -160,12 +184,20 @@ public static ColumnMappings parseColumnsMapping( columnMapping.qualifierName = null; columnMapping.qualifierNameBytes = null; columnMapping.hbaseRowKey = true; + } else if (colInfo.equals(HBASE_TIMESTAMP_COL)) { + timestampIndex = i; + columnMapping.familyName = colInfo; + columnMapping.familyNameBytes = Bytes.toBytes(colInfo); + columnMapping.qualifierName = null; + columnMapping.qualifierNameBytes = null; + columnMapping.hbaseTimestamp = true; } else { String [] parts = colInfo.split(":"); assert(parts.length > 0 && parts.length <= 2); columnMapping.familyName = parts[0]; columnMapping.familyNameBytes = Bytes.toBytes(parts[0]); columnMapping.hbaseRowKey = false; + columnMapping.hbaseTimestamp = false; if (parts.length == 2) { @@ -205,7 +237,7 @@ public static ColumnMappings parseColumnsMapping( columnsMapping.add(0, columnMapping); } - return new ColumnMappings(columnsMapping, rowKeyIndex); + return new ColumnMappings(columnsMapping, rowKeyIndex, timestampIndex); } public LazySimpleSerDe.SerDeParameters getSerdeParams() { @@ -228,7 +260,7 @@ public Object deserialize(Writable result) throws SerDeException { throw new SerDeException(getClass().getName() + ": expects ResultWritable!"); } - cachedHBaseRow.init(((ResultWritable) result).getResult(), serdeParams.getColumnMappings()); + cachedHBaseRow.init(((ResultWritable) result).getResult()); return cachedHBaseRow; } diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDeParameters.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDeParameters.java index 9efa494..a43520c 100644 --- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDeParameters.java +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDeParameters.java @@ -38,6 +38,8 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.util.ReflectionUtils; +import javax.annotation.Nullable; + /** * HBaseSerDeParameters encapsulates SerDeParameters and additional configurations that are specific for * HBaseSerDe. @@ -128,6 +130,14 @@ public ColumnMapping getKeyColumnMapping() { return columnMappings.getKeyMapping(); } + public int getTimestampIndex() { + return columnMappings.getTimestampIndex(); + } + + public ColumnMapping getTimestampColumnMapping() { + return columnMappings.getTimestampMapping(); + } + public ColumnMappings getColumnMappings() { return columnMappings; } @@ -175,17 +185,25 @@ private static HBaseKeyFactory createKeyFactory(Configuration job, Properties tb throws Exception { String factoryClassName = tbl.getProperty(HBaseSerDe.HBASE_COMPOSITE_KEY_FACTORY); if (factoryClassName != null) { - Class factoryClazz = Class.forName(factoryClassName); + Class factoryClazz = loadClass(factoryClassName, job); return (HBaseKeyFactory) ReflectionUtils.newInstance(factoryClazz, job); } String keyClassName = tbl.getProperty(HBaseSerDe.HBASE_COMPOSITE_KEY_CLASS); if (keyClassName != null) { - Class keyClass = Class.forName(keyClassName); + Class keyClass = loadClass(keyClassName, job); return new CompositeHBaseKeyFactory(keyClass); } return new DefaultHBaseKeyFactory(); } + private static Class loadClass(String className, @Nullable Configuration configuration) + throws Exception { + if (configuration != null) { + return configuration.getClassByName(className); + } + return Class.forName(className); + } + private List initValueFactories(Configuration conf, Properties tbl) throws SerDeException { List valueFactories = createValueFactories(conf, tbl); @@ -216,7 +234,7 @@ private static HBaseKeyFactory createKeyFactory(Configuration job, Properties tb + " must be set for hbase columns of type [" + STRUCT_SERIALIZATION_TYPE + "]"); } - Class structValueClass = job.getClassByName(structValueClassName); + Class structValueClass = loadClass(structValueClassName, job); valueFactories.add(new StructHBaseValueFactory(i, structValueClass)); } else { valueFactories.add(new DefaultHBaseValueFactory(i)); @@ -321,6 +339,10 @@ private Schema getSchema(Configuration conf, Properties tbl, ColumnMapping colMa tbl.getProperty(colMap.familyName + "." + qualifierName + "." + AvroSerdeUtils.SCHEMA_URL); } + if (serType == null) { + throw new IllegalArgumentException("serialization.type property is missing"); + } + String avroSchemaRetClass = tbl.getProperty(AvroSerdeUtils.SCHEMA_RETRIEVER); if (schemaLiteral == null && serClassName == null && schemaUrl == null @@ -333,7 +355,7 @@ private Schema getSchema(Configuration conf, Properties tbl, ColumnMapping colMa Class deserializerClass = null; if (serClassName != null) { - deserializerClass = conf.getClassByName(serClassName); + deserializerClass = loadClass(serClassName, conf); } Schema schema = null; @@ -354,4 +376,4 @@ private Schema getSchema(Configuration conf, Properties tbl, ColumnMapping colMa return schema; } -} \ No newline at end of file +} diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java index feb3cd1..814c86c 100644 --- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java @@ -186,7 +186,7 @@ public void preCreateTable(Table tbl) throws MetaException { Set uniqueColumnFamilies = new HashSet(); for (ColumnMapping colMap : columnMappings) { - if (!colMap.hbaseRowKey) { + if (!colMap.hbaseRowKey && !colMap.hbaseTimestamp) { uniqueColumnFamilies.add(colMap.familyName); } } @@ -213,7 +213,7 @@ public void preCreateTable(Table tbl) throws MetaException { for (ColumnMapping colMap : columnMappings) { - if (colMap.hbaseRowKey) { + if (colMap.hbaseRowKey || colMap.hbaseTimestamp) { continue; } @@ -475,6 +475,11 @@ public void configureJobConf(TableDesc tableDesc, JobConf jobConf) { TableMapReduceUtil.addDependencyJars(copy); merged.addAll(copy.getConfiguration().getStringCollection("tmpjars")); jobConf.set("tmpjars", StringUtils.arrayToString(merged.toArray(new String[0]))); + + // Get credentials using the configuration instance which has HBase properties + JobConf hbaseJobConf = new JobConf(getConf()); + org.apache.hadoop.hbase.mapred.TableMapReduceUtil.initCredentials(hbaseJobConf); + jobConf.getCredentials().mergeAll(hbaseJobConf.getCredentials()); } catch (Exception e) { throw new RuntimeException(e); } @@ -495,34 +500,38 @@ public static DecomposedPredicate decomposePredicate( HBaseSerDe hBaseSerDe, ExprNodeDesc predicate) { ColumnMapping keyMapping = hBaseSerDe.getHBaseSerdeParam().getKeyColumnMapping(); + ColumnMapping tsMapping = hBaseSerDe.getHBaseSerdeParam().getTimestampColumnMapping(); IndexPredicateAnalyzer analyzer = HiveHBaseTableInputFormat.newIndexPredicateAnalyzer( - keyMapping.columnName, keyMapping.columnType, keyMapping.binaryStorage.get(0)); - List searchConditions = - new ArrayList(); + keyMapping.columnName, keyMapping.isComparable(), + tsMapping == null ? null : tsMapping.columnName); + List conditions = new ArrayList(); ExprNodeGenericFuncDesc residualPredicate = - (ExprNodeGenericFuncDesc)analyzer.analyzePredicate(predicate, searchConditions); - int scSize = searchConditions.size(); - if (scSize < 1 || 2 < scSize) { - // Either there was nothing which could be pushed down (size = 0), - // there were complex predicates which we don't support yet. - // Currently supported are one of the form: - // 1. key < 20 (size = 1) - // 2. key = 20 (size = 1) - // 3. key < 20 and key > 10 (size = 2) - return null; - } - if (scSize == 2 && - (searchConditions.get(0).getComparisonOp() - .equals("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual") || - searchConditions.get(1).getComparisonOp() - .equals("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual"))) { - // If one of the predicates is =, then any other predicate with it is illegal. - return null; + (ExprNodeGenericFuncDesc)analyzer.analyzePredicate(predicate, conditions); + + for (List searchConditions: + HiveHBaseInputFormatUtil.decompose(conditions).values()) { + int scSize = searchConditions.size(); + if (scSize < 1 || 2 < scSize) { + // Either there was nothing which could be pushed down (size = 0), + // there were complex predicates which we don't support yet. + // Currently supported are one of the form: + // 1. key < 20 (size = 1) + // 2. key = 20 (size = 1) + // 3. key < 20 and key > 10 (size = 2) + return null; + } + if (scSize == 2 && + (searchConditions.get(0).getComparisonOp() + .equals("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual") || + searchConditions.get(1).getComparisonOp() + .equals("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual"))) { + // If one of the predicates is =, then any other predicate with it is illegal. + return null; + } } DecomposedPredicate decomposedPredicate = new DecomposedPredicate(); - decomposedPredicate.pushedPredicate = analyzer.translateSearchConditions( - searchConditions); + decomposedPredicate.pushedPredicate = analyzer.translateSearchConditions(conditions); decomposedPredicate.residualPredicate = residualPredicate; return decomposedPredicate; } diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseInputFormatUtil.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseInputFormatUtil.java index 5aa1d79..0524572 100644 --- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseInputFormatUtil.java +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseInputFormatUtil.java @@ -23,13 +23,16 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hive.hbase.ColumnMappings.ColumnMapping; +import org.apache.hadoop.hive.ql.index.IndexSearchCondition; import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.mapred.JobConf; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; /** * Util code common between HiveHBaseTableInputFormat and HiveHBaseTableSnapshotInputFormat. @@ -74,7 +77,7 @@ public static Scan getScan(JobConf jobConf) throws IOException { ColumnMapping[] columnsMapping = columnMappings.getColumnsMapping(); for (int i : readColIDs) { ColumnMapping colMap = columnsMapping[i]; - if (colMap.hbaseRowKey) { + if (colMap.hbaseRowKey || colMap.hbaseTimestamp) { continue; } @@ -99,7 +102,7 @@ public static Scan getScan(JobConf jobConf) throws IOException { // tables column projection. if (empty) { for (ColumnMapping colMap: columnMappings) { - if (colMap.hbaseRowKey) { + if (colMap.hbaseRowKey || colMap.hbaseTimestamp) { continue; } @@ -153,4 +156,19 @@ public static boolean getStorageFormatOfKey(String spec, String defaultFormat) t throw new IOException("Malformed string: " + spec); } } + + public static Map> decompose( + List searchConditions) { + Map> result = + new HashMap>(); + for (IndexSearchCondition condition : searchConditions) { + List conditions = result.get(condition.getColumnDesc().getColumn()); + if (conditions == null) { + conditions = new ArrayList(); + result.put(condition.getColumnDesc().getColumn(), conditions); + } + conditions.add(condition); + } + return result; + } } diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java index 4ac0803..2f18158 100644 --- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java @@ -20,7 +20,9 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -51,9 +53,11 @@ import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; import org.apache.hadoop.hive.serde2.lazy.LazyUtils; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.io.BooleanWritable; import org.apache.hadoop.io.FloatWritable; @@ -69,6 +73,7 @@ import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.hadoop.security.UserGroupInformation; /** * HiveHBaseTableInputFormat implements InputFormat for HBase storage handler @@ -171,7 +176,7 @@ public boolean next(ImmutableBytesWritable rowKey, ResultWritable value) throws * * @return converted table split if any */ - private Scan createFilterScan(JobConf jobConf, int iKey, boolean isKeyBinary) + private Scan createFilterScan(JobConf jobConf, int iKey, int iTimestamp, boolean isKeyBinary) throws IOException { // TODO: assert iKey is HBaseSerDe#HBASE_KEY_COL @@ -193,22 +198,29 @@ private Scan createFilterScan(JobConf jobConf, int iKey, boolean isKeyBinary) if (filterExprSerialized == null) { return scan; } + ExprNodeGenericFuncDesc filterExpr = Utilities.deserializeExpression(filterExprSerialized); - String colName = jobConf.get(serdeConstants.LIST_COLUMNS).split(",")[iKey]; + String keyColName = jobConf.get(serdeConstants.LIST_COLUMNS).split(",")[iKey]; String colType = jobConf.get(serdeConstants.LIST_COLUMN_TYPES).split(",")[iKey]; - IndexPredicateAnalyzer analyzer = newIndexPredicateAnalyzer(colName,colType, isKeyBinary); + boolean isKeyComparable = isKeyBinary || colType.equalsIgnoreCase("string"); + + String tsColName = null; + if (iTimestamp >= 0) { + tsColName = jobConf.get(serdeConstants.LIST_COLUMNS).split(",")[iTimestamp]; + } - List searchConditions = - new ArrayList(); - ExprNodeDesc residualPredicate = - analyzer.analyzePredicate(filterExpr, searchConditions); + IndexPredicateAnalyzer analyzer = + newIndexPredicateAnalyzer(keyColName, isKeyComparable, tsColName); + + List conditions = new ArrayList(); + ExprNodeDesc residualPredicate = analyzer.analyzePredicate(filterExpr, conditions); // There should be no residual since we already negotiated that earlier in // HBaseStorageHandler.decomposePredicate. However, with hive.optimize.index.filter // OpProcFactory#pushFilterToStorageHandler pushes the original filter back down again. - // Since pushed-down filters are not ommitted at the higher levels (and thus the + // Since pushed-down filters are not omitted at the higher levels (and thus the // contract of negotiation is ignored anyway), just ignore the residuals. // Re-assess this when negotiation is honored and the duplicate evaluation is removed. // THIS IGNORES RESIDUAL PARSING FROM HBaseStorageHandler#decomposePredicate @@ -216,9 +228,23 @@ private Scan createFilterScan(JobConf jobConf, int iKey, boolean isKeyBinary) LOG.debug("Ignoring residual predicate " + residualPredicate.getExprString()); } + Map> split = HiveHBaseInputFormatUtil.decompose(conditions); + List keyConditions = split.get(keyColName); + if (keyConditions != null && !keyConditions.isEmpty()) { + setupKeyRange(scan, keyConditions, isKeyBinary); + } + List tsConditions = split.get(tsColName); + if (tsConditions != null && !tsConditions.isEmpty()) { + setupTimeRange(scan, tsConditions); + } + return scan; + } + + private void setupKeyRange(Scan scan, List conditions, boolean isBinary) + throws IOException { // Convert the search condition into a restriction on the HBase scan byte [] startRow = HConstants.EMPTY_START_ROW, stopRow = HConstants.EMPTY_END_ROW; - for (IndexSearchCondition sc : searchConditions){ + for (IndexSearchCondition sc : conditions) { ExprNodeConstantEvaluator eval = new ExprNodeConstantEvaluator(sc.getConstantDesc()); PrimitiveObjectInspector objInspector; @@ -234,7 +260,7 @@ private Scan createFilterScan(JobConf jobConf, int iKey, boolean isKeyBinary) throw new IOException(e); } - byte [] constantVal = getConstantVal(writable, objInspector, isKeyBinary); + byte[] constantVal = getConstantVal(writable, objInspector, isBinary); String comparisonOp = sc.getComparisonOp(); if("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual".equals(comparisonOp)){ @@ -261,7 +287,52 @@ private Scan createFilterScan(JobConf jobConf, int iKey, boolean isKeyBinary) if (LOG.isDebugEnabled()) { LOG.debug(Bytes.toStringBinary(startRow) + " ~ " + Bytes.toStringBinary(stopRow)); } - return scan; + } + + private void setupTimeRange(Scan scan, List conditions) + throws IOException { + long start = 0; + long end = Long.MAX_VALUE; + for (IndexSearchCondition sc : conditions) { + long timestamp = getTimestampVal(sc); + String comparisonOp = sc.getComparisonOp(); + if("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual".equals(comparisonOp)){ + start = timestamp; + end = timestamp + 1; + } else if ("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan".equals(comparisonOp)){ + end = timestamp; + } else if ("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrGreaterThan" + .equals(comparisonOp)) { + start = timestamp; + } else if ("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPGreaterThan" + .equals(comparisonOp)){ + start = timestamp + 1; + } else if ("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrLessThan" + .equals(comparisonOp)){ + end = timestamp + 1; + } else { + throw new IOException(comparisonOp + " is not a supported comparison operator"); + } + } + scan.setTimeRange(start, end); + } + + private long getTimestampVal(IndexSearchCondition sc) throws IOException { + long timestamp; + try { + ExprNodeConstantEvaluator eval = new ExprNodeConstantEvaluator(sc.getConstantDesc()); + ObjectInspector inspector = eval.initialize(null); + Object value = eval.evaluate(null); + if (inspector instanceof LongObjectInspector) { + timestamp = ((LongObjectInspector)inspector).get(value); + } else { + PrimitiveObjectInspector primitive = (PrimitiveObjectInspector) inspector; + timestamp = PrimitiveObjectInspectorUtils.getTimestamp(value, primitive).getTime(); + } + } catch (HiveException e) { + throw new IOException(e); + } + return timestamp; } private byte[] getConstantVal(Object writable, PrimitiveObjectInspector poi, @@ -312,11 +383,6 @@ private Scan createFilterScan(JobConf jobConf, int iKey, boolean isKeyBinary) return next; } - static IndexPredicateAnalyzer newIndexPredicateAnalyzer( - String keyColumnName, TypeInfo keyColType, boolean isKeyBinary) { - return newIndexPredicateAnalyzer(keyColumnName, keyColType.getTypeName(), isKeyBinary); - } - /** * Instantiates a new predicate analyzer suitable for * determining how to push a filter down into the HBase scan, @@ -327,26 +393,34 @@ static IndexPredicateAnalyzer newIndexPredicateAnalyzer( * @return preconfigured predicate analyzer */ static IndexPredicateAnalyzer newIndexPredicateAnalyzer( - String keyColumnName, String keyColType, boolean isKeyBinary) { + String keyColumnName, boolean isKeyComparable, String timestampColumn) { IndexPredicateAnalyzer analyzer = new IndexPredicateAnalyzer(); // We can always do equality predicate. Just need to make sure we get appropriate // BA representation of constant of filter condition. - analyzer.addComparisonOp("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual"); // We can do other comparisons only if storage format in hbase is either binary - // or we are dealing with string types since there lexographic ordering will suffice. - if(isKeyBinary || (keyColType.equalsIgnoreCase("string"))){ - analyzer.addComparisonOp("org.apache.hadoop.hive.ql.udf.generic." + - "GenericUDFOPEqualOrGreaterThan"); - analyzer.addComparisonOp("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrLessThan"); - analyzer.addComparisonOp("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan"); - analyzer.addComparisonOp("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPGreaterThan"); + // or we are dealing with string types since there lexicographic ordering will suffice. + if (isKeyComparable) { + analyzer.addComparisonOp(keyColumnName, + "org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual", + "org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrGreaterThan", + "org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrLessThan", + "org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan", + "org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPGreaterThan"); + } else { + analyzer.addComparisonOp(keyColumnName, + "org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual"); } - // and only on the key column - analyzer.clearAllowedColumnNames(); - analyzer.allowColumnName(keyColumnName); + if (timestampColumn != null) { + analyzer.addComparisonOp(timestampColumn, + "org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual", + "org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrGreaterThan", + "org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrLessThan", + "org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan", + "org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPGreaterThan"); + } return analyzer; } @@ -355,7 +429,9 @@ static IndexPredicateAnalyzer newIndexPredicateAnalyzer( public InputSplit[] getSplits(JobConf jobConf, int numSplits) throws IOException { //obtain delegation tokens for the job - TableMapReduceUtil.initCredentials(jobConf); + if (UserGroupInformation.getCurrentUser().hasKerberosCredentials()) { + TableMapReduceUtil.initCredentials(jobConf); + } String hbaseTableName = jobConf.get(HBaseSerDe.HBASE_TABLE_NAME); setHTable(new HTable(HBaseConfiguration.create(jobConf), Bytes.toBytes(hbaseTableName))); @@ -374,6 +450,7 @@ static IndexPredicateAnalyzer newIndexPredicateAnalyzer( } int iKey = columnMappings.getKeyIndex(); + int iTimestamp = columnMappings.getTimestampIndex(); ColumnMapping keyMapping = columnMappings.getKeyMapping(); // Take filter pushdown into account while calculating splits; this @@ -382,7 +459,7 @@ static IndexPredicateAnalyzer newIndexPredicateAnalyzer( // split per region, the implementation actually takes the scan // definition into account and excludes regions which don't satisfy // the start/stop row conditions (HBASE-1829). - Scan scan = createFilterScan(jobConf, iKey, + Scan scan = createFilterScan(jobConf, iKey, iTimestamp, HiveHBaseInputFormatUtil.getStorageFormatOfKey(keyMapping.mappingSpec, jobConf.get(HBaseSerDe.HBASE_TABLE_DEFAULT_STORAGE_TYPE, "string"))); @@ -392,7 +469,7 @@ static IndexPredicateAnalyzer newIndexPredicateAnalyzer( // REVIEW: are we supposed to be applying the getReadColumnIDs // same as in getRecordReader? for (ColumnMapping colMap : columnMappings) { - if (colMap.hbaseRowKey) { + if (colMap.hbaseRowKey || colMap.hbaseTimestamp) { continue; } diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableOutputFormat.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableOutputFormat.java index e5ac9d3..3100885 100644 --- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableOutputFormat.java +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableOutputFormat.java @@ -42,6 +42,7 @@ import org.apache.hadoop.mapreduce.OutputCommitter; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.util.Progressable; +import org.apache.hadoop.security.UserGroupInformation; /** * HiveHBaseTableOutputFormat implements HiveOutputFormat for HBase tables. @@ -72,7 +73,9 @@ public void checkOutputSpecs(FileSystem fs, JobConf jc) throws IOException { //obtain delegation tokens for the job - TableMapReduceUtil.initCredentials(jc); + if (UserGroupInformation.getCurrentUser().hasKerberosCredentials()) { + TableMapReduceUtil.initCredentials(jc); + } String hbaseTableName = jc.get(HBaseSerDe.HBASE_TABLE_NAME); jc.set(TableOutputFormat.OUTPUT_TABLE, hbaseTableName); diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java index 6ac8423..8a3eac7 100644 --- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java @@ -22,13 +22,17 @@ import java.util.Arrays; import java.util.List; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hive.hbase.ColumnMappings.ColumnMapping; import org.apache.hadoop.hive.hbase.struct.HBaseValueFactory; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef; +import org.apache.hadoop.hive.serde2.lazy.LazyFactory; +import org.apache.hadoop.hive.serde2.lazy.LazyLong; import org.apache.hadoop.hive.serde2.lazy.LazyObjectBase; import org.apache.hadoop.hive.serde2.lazy.LazyStruct; +import org.apache.hadoop.hive.serde2.lazy.LazyTimestamp; import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazySimpleStructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; @@ -42,43 +46,48 @@ * The HBase columns mapping of the row. */ private Result result; - private ColumnMapping[] columnsMapping; private ArrayList cachedList; - private final int iKey; private final HBaseKeyFactory keyFactory; private final List valueFactories; + private final ColumnMapping[] columnsMapping; - public LazyHBaseRow(LazySimpleStructObjectInspector oi) { - this(oi, -1, null, null); + @VisibleForTesting + LazyHBaseRow(LazySimpleStructObjectInspector oi, ColumnMappings columnMappings) { + super(oi); + this.keyFactory = DefaultHBaseKeyFactory.forTest(null, columnMappings); + this.valueFactories = null; + this.columnsMapping = columnMappings.getColumnsMapping(); } /** * Construct a LazyHBaseRow object with the ObjectInspector. */ - public LazyHBaseRow(LazySimpleStructObjectInspector oi, int iKey, HBaseKeyFactory keyFactory, - List valueFactories) { + public LazyHBaseRow(LazySimpleStructObjectInspector oi, HBaseSerDeParameters serdeParams) { super(oi); - this.iKey = iKey; - this.keyFactory = keyFactory; - this.valueFactories = valueFactories; + this.keyFactory = serdeParams.getKeyFactory(); + this.valueFactories = serdeParams.getValueFactories(); + this.columnsMapping = serdeParams.getColumnMappings().getColumnsMapping(); } /** * Set the HBase row data(a Result writable) for this LazyStruct. * @see LazyHBaseRow#init(org.apache.hadoop.hbase.client.Result) */ - public void init(Result r, ColumnMappings columnsMappings) { + public void init(Result r) { this.result = r; - this.columnsMapping = columnsMappings.getColumnsMapping(); setParsed(false); } @Override - protected LazyObjectBase createLazyField(int fieldID, StructField fieldRef) throws SerDeException { - if (fieldID == iKey) { + protected LazyObjectBase createLazyField(final int fieldID, final StructField fieldRef) + throws SerDeException { + if (columnsMapping[fieldID].hbaseRowKey) { return keyFactory.createKey(fieldRef.getFieldObjectInspector()); } + if (columnsMapping[fieldID].hbaseTimestamp) { + return LazyFactory.createLazyObject(fieldRef.getFieldObjectInspector()); + } if (valueFactories != null) { return valueFactories.get(fieldID).createValueObject(fieldRef.getFieldObjectInspector()); @@ -121,7 +130,6 @@ private void initFields() { * Get the field out of the row without checking whether parsing is needed. * This is called by both getField and getFieldsAsList. * @param fieldID The id of the field starting from 0. - * @param nullSequence The sequence representing NULL value. * @return The value of the field */ private Object uncheckedGetField(int fieldID) { @@ -136,6 +144,14 @@ private Object uncheckedGetField(int fieldID) { if (colMap.hbaseRowKey) { ref = new ByteArrayRef(); ref.setData(result.getRow()); + } else if (colMap.hbaseTimestamp) { + long timestamp = result.rawCells()[0].getTimestamp(); // from hbase-0.96.0 + LazyObjectBase lz = fields[fieldID]; + if (lz instanceof LazyTimestamp) { + ((LazyTimestamp) lz).getWritableObject().setTime(timestamp); + } else { + ((LazyLong) lz).getWritableObject().set(timestamp); + } } else { if (colMap.qualifierName == null) { // it is a column family diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/DefaultHBaseValueFactory.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/DefaultHBaseValueFactory.java index f7a425d..e60b844 100644 --- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/DefaultHBaseValueFactory.java +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/DefaultHBaseValueFactory.java @@ -35,7 +35,7 @@ /** * Default implementation of the {@link HBaseValueFactory} * */ -public class DefaultHBaseValueFactory implements HBaseValueFactory{ +public class DefaultHBaseValueFactory implements HBaseValueFactory { protected LazySimpleSerDe.SerDeParameters serdeParams; protected ColumnMappings columnMappings; @@ -49,32 +49,32 @@ public DefaultHBaseValueFactory(int fieldID) { this.fieldID = fieldID; } - @Override + @Override public void init(HBaseSerDeParameters hbaseParams, Configuration conf, Properties properties) - throws SerDeException { + throws SerDeException { this.hbaseParams = hbaseParams; this.serdeParams = hbaseParams.getSerdeParams(); this.columnMappings = hbaseParams.getColumnMappings(); this.properties = properties; this.conf = conf; - } + } - @Override - public ObjectInspector createValueObjectInspector(TypeInfo type) - throws SerDeException { + @Override + public ObjectInspector createValueObjectInspector(TypeInfo type) + throws SerDeException { return LazyFactory.createLazyObjectInspector(type, serdeParams.getSeparators(), 1, serdeParams.getNullSequence(), serdeParams.isEscaped(), serdeParams.getEscapeChar()); - } + } @Override public LazyObjectBase createValueObject(ObjectInspector inspector) throws SerDeException { return HBaseSerDeHelper.createLazyField(columnMappings.getColumnsMapping(), fieldID, inspector); } - @Override - public byte[] serializeValue(Object object, StructField field) - throws IOException { + @Override + public byte[] serializeValue(Object object, StructField field) + throws IOException { // TODO Add support for serialization of values here - return null; - } -} \ No newline at end of file + return null; + } +} diff --git a/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseObject.java b/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseObject.java index 9a31f0f..b2bdd19 100644 --- a/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseObject.java +++ b/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseObject.java @@ -459,7 +459,7 @@ public void testLazyHBaseRow1() throws SerDeException { List fieldTypeInfos = TypeInfoUtils.getTypeInfosFromTypeString( "string,int,array,map,string"); - List fieldNames = Arrays.asList(new String[]{"key", "a", "b", "c", "d"}); + List fieldNames = Arrays.asList("key", "a", "b", "c", "d"); Text nullSequence = new Text("\\N"); String hbaseColsMapping = ":key,cfa:a,cfa:b,cfb:c,cfb:d"; @@ -483,7 +483,7 @@ public void testLazyHBaseRow1() throws SerDeException { ObjectInspector oi = LazyFactory.createLazyStructInspector(fieldNames, fieldTypeInfos, new byte[] {' ', ':', '='}, nullSequence, false, false, (byte)0); - LazyHBaseRow o = new LazyHBaseRow((LazySimpleStructObjectInspector) oi); + LazyHBaseRow o = new LazyHBaseRow((LazySimpleStructObjectInspector) oi, columnMappings); List kvs = new ArrayList(); @@ -497,7 +497,7 @@ public void testLazyHBaseRow1() throws SerDeException { Bytes.toBytes("cfb"), Bytes.toBytes("d"), Bytes.toBytes("hi"))); Result r = new Result(kvs); - o.init(r, columnMappings); + o.init(r); assertEquals( ("{'key':'test-row','a':123,'b':['a','b','c']," @@ -511,7 +511,7 @@ public void testLazyHBaseRow1() throws SerDeException { Bytes.toBytes("cfb"), Bytes.toBytes("c"), Bytes.toBytes("d=e:f=g"))); r = new Result(kvs); - o.init(r, columnMappings); + o.init(r); assertEquals( ("{'key':'test-row','a':123,'b':null," @@ -527,7 +527,7 @@ public void testLazyHBaseRow1() throws SerDeException { Bytes.toBytes("cfb"), Bytes.toBytes("d"), Bytes.toBytes("no"))); r = new Result(kvs); - o.init(r, columnMappings); + o.init(r); assertEquals( ("{'key':'test-row','a':null,'b':['a']," @@ -541,7 +541,7 @@ public void testLazyHBaseRow1() throws SerDeException { Bytes.toBytes("cfb"), Bytes.toBytes("d"), Bytes.toBytes("no"))); r = new Result(kvs); - o.init(r, columnMappings); + o.init(r); assertEquals( ("{'key':'test-row','a':null,'b':['','a','','']," @@ -565,7 +565,7 @@ public void testLazyHBaseRow1() throws SerDeException { Bytes.toBytes("cfb"), Bytes.toBytes("d"), Bytes.toBytes(""))); r = new Result(kvs); - o.init(r, columnMappings); + o.init(r); assertEquals( "{'key':'test-row','a':123,'b':[],'c':{},'d':''}".replace("'", "\""), @@ -609,7 +609,7 @@ public void testLazyHBaseRow2() throws SerDeException { fieldTypeInfos, new byte[] {' ', ':', '='}, nullSequence, false, false, (byte) 0); - LazyHBaseRow o = new LazyHBaseRow((LazySimpleStructObjectInspector) oi); + LazyHBaseRow o = new LazyHBaseRow((LazySimpleStructObjectInspector) oi, columnMappings); List kvs = new ArrayList(); kvs.add(new KeyValue(Bytes.toBytes("test-row"), @@ -624,7 +624,7 @@ public void testLazyHBaseRow2() throws SerDeException { Bytes.toBytes("cfc"), Bytes.toBytes("d"), Bytes.toBytes("hi"))); Result r = new Result(kvs); - o.init(r, columnMappings); + o.init(r); assertEquals( ("{'key':'test-row','a':123,'b':['a','b','c']," @@ -640,7 +640,7 @@ public void testLazyHBaseRow2() throws SerDeException { Bytes.toBytes("cfb"), Bytes.toBytes("f"), Bytes.toBytes("g"))); r = new Result(kvs); - o.init(r, columnMappings); + o.init(r); assertEquals( ("{'key':'test-row','a':123,'b':null," @@ -656,7 +656,7 @@ public void testLazyHBaseRow2() throws SerDeException { Bytes.toBytes("cfc"), Bytes.toBytes("d"), Bytes.toBytes("no"))); r = new Result(kvs); - o.init(r, columnMappings); + o.init(r); assertEquals( ("{'key':'test-row','a':null,'b':['a']," @@ -670,7 +670,7 @@ public void testLazyHBaseRow2() throws SerDeException { Bytes.toBytes("cfc"), Bytes.toBytes("d"), Bytes.toBytes("no"))); r = new Result(kvs); - o.init(r, columnMappings); + o.init(r); assertEquals( ("{'key':'test-row','a':null,'b':['','a','','']," @@ -686,7 +686,7 @@ public void testLazyHBaseRow2() throws SerDeException { Bytes.toBytes("cfc"), Bytes.toBytes("d"), Bytes.toBytes(""))); r = new Result(kvs); - o.init(r, columnMappings); + o.init(r); assertEquals( "{'key':'test-row','a':123,'b':[],'c':{},'d':''}".replace("'", "\""), @@ -733,7 +733,7 @@ public void testLazyHBaseRow3() throws SerDeException { LazyFactory.createLazyStructInspector(fieldNames, fieldTypeInfos, new byte [] {' ', ':', '='}, nullSequence, false, false, (byte) 0); - LazyHBaseRow o = new LazyHBaseRow((LazySimpleStructObjectInspector) oi); + LazyHBaseRow o = new LazyHBaseRow((LazySimpleStructObjectInspector) oi, columnMappings); byte [] rowKey = "row-key".getBytes(); List kvs = new ArrayList(); @@ -785,7 +785,7 @@ public void testLazyHBaseRow3() throws SerDeException { Collections.sort(kvs, KeyValue.COMPARATOR); Result result = new Result(kvs); - o.init(result, columnMappings); + o.init(result); List fieldRefs = ((StructObjectInspector) oi).getAllStructFieldRefs(); diff --git a/hbase-handler/src/test/queries/positive/hbase_timestamp.q b/hbase-handler/src/test/queries/positive/hbase_timestamp.q new file mode 100644 index 0000000..6f1e205 --- /dev/null +++ b/hbase-handler/src/test/queries/positive/hbase_timestamp.q @@ -0,0 +1,41 @@ +DROP TABLE hbase_table; +CREATE TABLE hbase_table (key string, value string, time timestamp) + STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' + WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,cf:string,:timestamp"); +DESC extended hbase_table; +FROM src INSERT OVERWRITE TABLE hbase_table SELECT key, value, "2012-02-23 10:14:52" WHERE (key % 17) = 0; +SELECT * FROM hbase_table; + +DROP TABLE hbase_table; +CREATE TABLE hbase_table (key string, value string, time bigint) + STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' + WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,cf:string,:timestamp"); +FROM src INSERT OVERWRITE TABLE hbase_table SELECT key, value, 1329959754000 WHERE (key % 17) = 0; +SELECT key, value, cast(time as timestamp) FROM hbase_table; + +DROP TABLE hbase_table; +CREATE TABLE hbase_table (key string, value string, time bigint) + STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' + WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,cf:string,:timestamp"); +insert overwrite table hbase_table select key,value,ts FROM +( + select key, value, 100000000000 as ts from src WHERE (key % 33) = 0 + UNION ALL + select key, value, 200000000000 as ts from src WHERE (key % 37) = 0 +) T; + +explain +SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time < 200000000000; +SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time < 200000000000; + +explain +SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time > 100000000000; +SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time > 100000000000; + +explain +SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time <= 100000000000; +SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time <= 100000000000; + +explain +SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time >= 200000000000; +SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time >= 200000000000; diff --git a/hbase-handler/src/test/results/positive/hbase_pushdown.q.out b/hbase-handler/src/test/results/positive/hbase_pushdown.q.out index 520285a..8a979bf 100644 --- a/hbase-handler/src/test/results/positive/hbase_pushdown.q.out +++ b/hbase-handler/src/test/results/positive/hbase_pushdown.q.out @@ -94,7 +94,7 @@ STAGE PLANS: predicate: (value like '%90%') (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: key (type: int), value (type: string) + expressions: 90 (type: int), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator @@ -142,7 +142,7 @@ STAGE PLANS: predicate: (value like '%90%') (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: key (type: int), value (type: string) + expressions: 90 (type: int), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator @@ -194,7 +194,7 @@ STAGE PLANS: predicate: ((value like '%90%') and (key = UDFToInteger(value))) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: key (type: int), value (type: string) + expressions: 90 (type: int), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator diff --git a/hbase-handler/src/test/results/positive/hbase_queries.q.out b/hbase-handler/src/test/results/positive/hbase_queries.q.out index 1972236..b1e7936 100644 --- a/hbase-handler/src/test/results/positive/hbase_queries.q.out +++ b/hbase-handler/src/test/results/positive/hbase_queries.q.out @@ -271,7 +271,7 @@ STAGE PLANS: alias: hbase_table_2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Filter Operator - predicate: ((key < 120) and key is not null) (type: boolean) + predicate: (key < 120) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) @@ -287,7 +287,7 @@ STAGE PLANS: alias: hbase_table_1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Filter Operator - predicate: ((100 < key) and key is not null) (type: boolean) + predicate: (100 < key) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: key (type: int) diff --git a/hbase-handler/src/test/results/positive/hbase_timestamp.q.out b/hbase-handler/src/test/results/positive/hbase_timestamp.q.out new file mode 100644 index 0000000..f70d371 --- /dev/null +++ b/hbase-handler/src/test/results/positive/hbase_timestamp.q.out @@ -0,0 +1,357 @@ +PREHOOK: query: DROP TABLE hbase_table +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE hbase_table +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE hbase_table (key string, value string, time timestamp) + STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' + WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,cf:string,:timestamp") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@hbase_table +POSTHOOK: query: CREATE TABLE hbase_table (key string, value string, time timestamp) + STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' + WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,cf:string,:timestamp") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@hbase_table +PREHOOK: query: DESC extended hbase_table +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@hbase_table +POSTHOOK: query: DESC extended hbase_table +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@hbase_table +key string from deserializer +value string from deserializer +time timestamp from deserializer + +#### A masked pattern was here #### +PREHOOK: query: FROM src INSERT OVERWRITE TABLE hbase_table SELECT key, value, "2012-02-23 10:14:52" WHERE (key % 17) = 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@hbase_table +POSTHOOK: query: FROM src INSERT OVERWRITE TABLE hbase_table SELECT key, value, "2012-02-23 10:14:52" WHERE (key % 17) = 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@hbase_table +PREHOOK: query: SELECT * FROM hbase_table +PREHOOK: type: QUERY +PREHOOK: Input: default@hbase_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM hbase_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@hbase_table +#### A masked pattern was here #### +0 val_0 2012-02-23 10:14:52 +119 val_119 2012-02-23 10:14:52 +136 val_136 2012-02-23 10:14:52 +153 val_153 2012-02-23 10:14:52 +17 val_17 2012-02-23 10:14:52 +170 val_170 2012-02-23 10:14:52 +187 val_187 2012-02-23 10:14:52 +221 val_221 2012-02-23 10:14:52 +238 val_238 2012-02-23 10:14:52 +255 val_255 2012-02-23 10:14:52 +272 val_272 2012-02-23 10:14:52 +289 val_289 2012-02-23 10:14:52 +306 val_306 2012-02-23 10:14:52 +323 val_323 2012-02-23 10:14:52 +34 val_34 2012-02-23 10:14:52 +374 val_374 2012-02-23 10:14:52 +459 val_459 2012-02-23 10:14:52 +493 val_493 2012-02-23 10:14:52 +51 val_51 2012-02-23 10:14:52 +85 val_85 2012-02-23 10:14:52 +PREHOOK: query: DROP TABLE hbase_table +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@hbase_table +PREHOOK: Output: default@hbase_table +POSTHOOK: query: DROP TABLE hbase_table +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@hbase_table +POSTHOOK: Output: default@hbase_table +PREHOOK: query: CREATE TABLE hbase_table (key string, value string, time bigint) + STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' + WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,cf:string,:timestamp") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@hbase_table +POSTHOOK: query: CREATE TABLE hbase_table (key string, value string, time bigint) + STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' + WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,cf:string,:timestamp") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@hbase_table +PREHOOK: query: FROM src INSERT OVERWRITE TABLE hbase_table SELECT key, value, 1329959754000 WHERE (key % 17) = 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@hbase_table +POSTHOOK: query: FROM src INSERT OVERWRITE TABLE hbase_table SELECT key, value, 1329959754000 WHERE (key % 17) = 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@hbase_table +PREHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table +PREHOOK: type: QUERY +PREHOOK: Input: default@hbase_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@hbase_table +#### A masked pattern was here #### +0 val_0 2012-02-22 17:15:54 +119 val_119 2012-02-22 17:15:54 +136 val_136 2012-02-22 17:15:54 +153 val_153 2012-02-22 17:15:54 +17 val_17 2012-02-22 17:15:54 +170 val_170 2012-02-22 17:15:54 +187 val_187 2012-02-22 17:15:54 +221 val_221 2012-02-22 17:15:54 +238 val_238 2012-02-22 17:15:54 +255 val_255 2012-02-22 17:15:54 +272 val_272 2012-02-22 17:15:54 +289 val_289 2012-02-22 17:15:54 +306 val_306 2012-02-22 17:15:54 +323 val_323 2012-02-22 17:15:54 +34 val_34 2012-02-22 17:15:54 +374 val_374 2012-02-22 17:15:54 +459 val_459 2012-02-22 17:15:54 +493 val_493 2012-02-22 17:15:54 +51 val_51 2012-02-22 17:15:54 +85 val_85 2012-02-22 17:15:54 +PREHOOK: query: DROP TABLE hbase_table +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@hbase_table +PREHOOK: Output: default@hbase_table +POSTHOOK: query: DROP TABLE hbase_table +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@hbase_table +POSTHOOK: Output: default@hbase_table +PREHOOK: query: CREATE TABLE hbase_table (key string, value string, time bigint) + STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' + WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,cf:string,:timestamp") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@hbase_table +POSTHOOK: query: CREATE TABLE hbase_table (key string, value string, time bigint) + STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' + WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,cf:string,:timestamp") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@hbase_table +PREHOOK: query: insert overwrite table hbase_table select key,value,ts FROM +( + select key, value, 100000000000 as ts from src WHERE (key % 33) = 0 + UNION ALL + select key, value, 200000000000 as ts from src WHERE (key % 37) = 0 +) T +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@hbase_table +POSTHOOK: query: insert overwrite table hbase_table select key,value,ts FROM +( + select key, value, 100000000000 as ts from src WHERE (key % 33) = 0 + UNION ALL + select key, value, 200000000000 as ts from src WHERE (key % 37) = 0 +) T +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@hbase_table +PREHOOK: query: explain +SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time < 200000000000 +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time < 200000000000 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: hbase_table + filterExpr: (((key > 100) and (key < 400)) and (time < 200000000000)) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Filter Operator + predicate: (((key > 100) and (key < 400)) and (time < 200000000000)) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string), CAST( time AS TIMESTAMP) (type: timestamp) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time < 200000000000 +PREHOOK: type: QUERY +PREHOOK: Input: default@hbase_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time < 200000000000 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@hbase_table +#### A masked pattern was here #### +165 val_165 1973-03-03 01:46:40 +396 val_396 1973-03-03 01:46:40 +PREHOOK: query: explain +SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time > 100000000000 +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time > 100000000000 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: hbase_table + filterExpr: (((key > 100) and (key < 400)) and (time > 100000000000)) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Filter Operator + predicate: (((key > 100) and (key < 400)) and (time > 100000000000)) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string), CAST( time AS TIMESTAMP) (type: timestamp) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time > 100000000000 +PREHOOK: type: QUERY +PREHOOK: Input: default@hbase_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time > 100000000000 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@hbase_table +#### A masked pattern was here #### +111 val_111 1976-05-03 12:33:20 +222 val_222 1976-05-03 12:33:20 +296 val_296 1976-05-03 12:33:20 +333 val_333 1976-05-03 12:33:20 +PREHOOK: query: explain +SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time <= 100000000000 +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time <= 100000000000 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: hbase_table + filterExpr: (((key > 100) and (key < 400)) and (time <= 100000000000)) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Filter Operator + predicate: (((key > 100) and (key < 400)) and (time <= 100000000000)) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string), CAST( time AS TIMESTAMP) (type: timestamp) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time <= 100000000000 +PREHOOK: type: QUERY +PREHOOK: Input: default@hbase_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time <= 100000000000 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@hbase_table +#### A masked pattern was here #### +165 val_165 1973-03-03 01:46:40 +396 val_396 1973-03-03 01:46:40 +PREHOOK: query: explain +SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time >= 200000000000 +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time >= 200000000000 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: hbase_table + filterExpr: (((key > 100) and (key < 400)) and (time >= 200000000000)) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Filter Operator + predicate: (((key > 100) and (key < 400)) and (time >= 200000000000)) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string), CAST( time AS TIMESTAMP) (type: timestamp) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time >= 200000000000 +PREHOOK: type: QUERY +PREHOOK: Input: default@hbase_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time >= 200000000000 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@hbase_table +#### A masked pattern was here #### +111 val_111 1976-05-03 12:33:20 +222 val_222 1976-05-03 12:33:20 +296 val_296 1976-05-03 12:33:20 +333 val_333 1976-05-03 12:33:20 diff --git a/hcatalog/core/pom.xml b/hcatalog/core/pom.xml index 42cb907..8acb02f 100644 --- a/hcatalog/core/pom.xml +++ b/hcatalog/core/pom.xml @@ -25,7 +25,7 @@ org.apache.hive.hcatalog hive-hcatalog - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../pom.xml diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/CreateTableHook.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/CreateTableHook.java index 172ff01..8b69223 100644 --- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/CreateTableHook.java +++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/CreateTableHook.java @@ -26,7 +26,6 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.exec.DDLTask; import org.apache.hadoop.hive.ql.exec.Task; @@ -195,8 +194,7 @@ public void postAnalyze(HiveSemanticAnalyzerHookContext context, //authorize against the table operation so that location permissions can be checked if any - if (HiveConf.getBoolVar(context.getConf(), - HiveConf.ConfVars.HIVE_AUTHORIZATION_ENABLED)) { + if (HCatAuthUtil.isAuthorizationEnabled(context.getConf())) { authorize(table, Privilege.CREATE); } } catch (HiveException ex) { diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/HCatAuthUtil.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/HCatAuthUtil.java new file mode 100644 index 0000000..6dce9c4 --- /dev/null +++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/HCatAuthUtil.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hive.hcatalog.cli.SemanticAnalysis; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.session.SessionState; + +final class HCatAuthUtil { + public static boolean isAuthorizationEnabled(Configuration conf) { + // the session state getAuthorizer can return null even if authorization is + // enabled if the V2 api of authorizer in use. + // The additional authorization checks happening in hcatalog are designed to + // work with storage based authorization (on client side). It should not try doing + // additional checks if a V2 authorizer is in use. The reccomended configuration is to + // use storage based authorization in metastore server + return HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_AUTHORIZATION_ENABLED) + && SessionState.get().getAuthorizer() != null; + } +} diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/HCatSemanticAnalyzerBase.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/HCatSemanticAnalyzerBase.java index 0184dc0..5b3ef94 100644 --- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/HCatSemanticAnalyzerBase.java +++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/HCatSemanticAnalyzerBase.java @@ -22,7 +22,6 @@ import java.io.Serializable; import java.util.List; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.metadata.AuthorizationException; @@ -89,8 +88,7 @@ public void postAnalyze(HiveSemanticAnalyzerHookContext context, protected void authorizeDDL(HiveSemanticAnalyzerHookContext context, List> rootTasks) throws SemanticException { - if (!HiveConf.getBoolVar(context.getConf(), - HiveConf.ConfVars.HIVE_AUTHORIZATION_ENABLED)) { + if (!HCatAuthUtil.isAuthorizationEnabled(context.getConf())) { return; } diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java index 4fdb5c9..2f624df 100644 --- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java +++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java @@ -33,6 +33,7 @@ import java.util.Map; import java.util.Properties; +import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -120,25 +121,11 @@ public static Object deserialize(String str) throws IOException { } public static String encodeBytes(byte[] bytes) { - StringBuilder strBuf = new StringBuilder(); - - for (int i = 0; i < bytes.length; i++) { - strBuf.append((char) (((bytes[i] >> 4) & 0xF) + ('a'))); - strBuf.append((char) (((bytes[i]) & 0xF) + ('a'))); - } - - return strBuf.toString(); + return new String(Base64.encodeBase64(bytes, false, false)); } public static byte[] decodeBytes(String str) { - byte[] bytes = new byte[str.length() / 2]; - for (int i = 0; i < str.length(); i += 2) { - char c = str.charAt(i); - bytes[i / 2] = (byte) ((c - 'a') << 4); - c = str.charAt(i + 1); - bytes[i / 2] += (c - 'a'); - } - return bytes; + return Base64.decodeBase64(str.getBytes()); } public static List getHCatFieldSchemaList( diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/TestHCatAuthUtil.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/TestHCatAuthUtil.java new file mode 100644 index 0000000..830dcb8 --- /dev/null +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/TestHCatAuthUtil.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hive.hcatalog.cli.SemanticAnalysis; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.ql.security.HiveAuthenticationProvider; +import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthorizer; +import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthorizerFactory; +import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzPluginException; +import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzSessionContext; +import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveMetastoreClientFactory; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.junit.Test; +import org.mockito.Mockito; + +/** + * Test HCatAuthUtil + */ +public class TestHCatAuthUtil { + + public static class DummyV2AuthorizerFactory implements HiveAuthorizerFactory { + + @Override + public HiveAuthorizer createHiveAuthorizer(HiveMetastoreClientFactory metastoreClientFactory, + HiveConf conf, HiveAuthenticationProvider hiveAuthenticator, HiveAuthzSessionContext ctx) + throws HiveAuthzPluginException { + return Mockito.mock(HiveAuthorizer.class); + } + } + + /** + * Test with auth enabled and v1 auth + */ + @Test + public void authEnabledV1Auth() throws Exception { + HiveConf hcatConf = new HiveConf(this.getClass()); + hcatConf.setBoolVar(ConfVars.HIVE_AUTHORIZATION_ENABLED, true); + SessionState.start(hcatConf); + assertTrue("hcat auth should be enabled", HCatAuthUtil.isAuthorizationEnabled(hcatConf)); + } + + /** + * Test with auth enabled and v2 auth + */ + @Test + public void authEnabledV2Auth() throws Exception { + HiveConf hcatConf = new HiveConf(this.getClass()); + hcatConf.setBoolVar(ConfVars.HIVE_AUTHORIZATION_ENABLED, true); + hcatConf.setVar(ConfVars.HIVE_AUTHORIZATION_MANAGER, DummyV2AuthorizerFactory.class.getName()); + SessionState.start(hcatConf); + assertFalse("hcat auth should be disabled", HCatAuthUtil.isAuthorizationEnabled(hcatConf)); + } + + /** + * Test with auth disabled + */ + @Test + public void authDisabled() throws Exception { + HiveConf hcatConf = new HiveConf(this.getClass()); + hcatConf.setBoolVar(ConfVars.HIVE_AUTHORIZATION_ENABLED, false); + SessionState.start(hcatConf); + assertFalse("hcat auth should be disabled", HCatAuthUtil.isAuthorizationEnabled(hcatConf)); + } +} diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/schema/TestHCatSchemaUtils.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/schema/TestHCatSchemaUtils.java index 56c1844..e80c3d2 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/schema/TestHCatSchemaUtils.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/schema/TestHCatSchemaUtils.java @@ -35,10 +35,10 @@ public void testSimpleOperation() throws Exception { String typeString = "struct," + + "contact:struct," + "currently_registered_courses:array," + "current_grades:map," - + "phnos:array>,blah:array>"; + + "phNos:array>,blah:array>"; TypeInfo ti = TypeInfoUtils.getTypeInfoFromTypeString(typeString); @@ -46,8 +46,9 @@ public void testSimpleOperation() throws Exception { LOG.info("Type name : {}", ti.getTypeName()); LOG.info("HCatSchema : {}", hsch); assertEquals(hsch.size(), 1); - assertEquals(ti.getTypeName(), hsch.get(0).getTypeString()); - assertEquals(hsch.get(0).getTypeString(), typeString); + // Looks like HCatFieldSchema.getTypeString() lower-cases its results + assertEquals(ti.getTypeName().toLowerCase(), hsch.get(0).getTypeString()); + assertEquals(hsch.get(0).getTypeString(), typeString.toLowerCase()); } @SuppressWarnings("unused") diff --git a/hcatalog/hcatalog-pig-adapter/pom.xml b/hcatalog/hcatalog-pig-adapter/pom.xml index 7589efb..2d959e6 100644 --- a/hcatalog/hcatalog-pig-adapter/pom.xml +++ b/hcatalog/hcatalog-pig-adapter/pom.xml @@ -25,7 +25,7 @@ org.apache.hive.hcatalog hive-hcatalog - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../pom.xml diff --git a/hcatalog/pom.xml b/hcatalog/pom.xml index 4b75ef5..b393795 100644 --- a/hcatalog/pom.xml +++ b/hcatalog/pom.xml @@ -24,7 +24,7 @@ org.apache.hive hive - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../pom.xml @@ -46,6 +46,15 @@ streaming + + + org.mockito + mockito-all + ${mockito-all.version} + test + + + hadoop-1 diff --git a/hcatalog/server-extensions/pom.xml b/hcatalog/server-extensions/pom.xml index d222c59..14c0d52 100644 --- a/hcatalog/server-extensions/pom.xml +++ b/hcatalog/server-extensions/pom.xml @@ -25,7 +25,7 @@ org.apache.hive.hcatalog hive-hcatalog - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../pom.xml diff --git a/hcatalog/src/test/e2e/templeton/README.txt b/hcatalog/src/test/e2e/templeton/README.txt index bf2843c..d272985 100644 --- a/hcatalog/src/test/e2e/templeton/README.txt +++ b/hcatalog/src/test/e2e/templeton/README.txt @@ -101,7 +101,7 @@ For Hadoop 2.x you would need to upload hadoop-mapreduce-client-jobclient.jar to Also see https://cwiki.apache.org/confluence/display/Hive/WebHCat+InstallWebHCat#WebHCatInstallWebHCat-HadoopDistributedCache for notes on additional JAR files to copy to HDFS. -5. Make sure TEMPLETON_HOME evnironment variable is set +5. Make sure TEMPLETON_HOME environment variable is set 6. hadoop/conf/core-site.xml should have items described in https://cwiki.apache.org/confluence/display/Hive/WebHCat+InstallWebHCat#WebHCatInstallWebHCat-Permissions @@ -124,6 +124,9 @@ You may also need to adjust the following in Pig's build.xml as needed: true +8.Sqoop test require JDBC jar to be placed on HDFS for whichever DB the test is configured with, +for example mysql-connector-java-5.1.30-bin.jar. + **** **** See deployers/ for scripts that automate a lot of the set up. **** diff --git a/hcatalog/src/test/e2e/templeton/build.xml b/hcatalog/src/test/e2e/templeton/build.xml index 8bc363d..ee98a99 100644 --- a/hcatalog/src/test/e2e/templeton/build.xml +++ b/hcatalog/src/test/e2e/templeton/build.xml @@ -111,6 +111,7 @@ + @@ -148,6 +149,7 @@ + @@ -186,7 +188,9 @@ + + diff --git a/hcatalog/src/test/e2e/templeton/deployers/clean_file_system.sh b/hcatalog/src/test/e2e/templeton/deployers/clean_file_system.sh index be3bbf5..d1fd613 100755 --- a/hcatalog/src/test/e2e/templeton/deployers/clean_file_system.sh +++ b/hcatalog/src/test/e2e/templeton/deployers/clean_file_system.sh @@ -28,6 +28,6 @@ echo "Deleting artifacts from HDFS..." ${HADOOP_HOME}/bin/hdfs dfs -rm -r /user/hive/ /user/${USER}/ /user/templeton /apps /tmp /sqoopoutputdir -${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /tmp/hadoop-${USER} /user/hive/warehouse /user/${USER}/ /user/templeton /apps/templeton /tmp/hadoop-yarn /tmp/templeton_test_out +${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /tmp/hadoop-${USER} /user/hive/warehouse /user/${USER}/ /user/templeton /apps/templeton/jdbc /tmp/hadoop-yarn /tmp/templeton_test_out ${HADOOP_HOME}/bin/hdfs dfs -chmod -R a+rwx /user /tmp/ ${HADOOP_HOME}/bin/hdfs dfs -chmod g+rwx /user/hive/warehouse diff --git a/hcatalog/src/test/e2e/templeton/deployers/config/webhcat/webhcat-site.xml b/hcatalog/src/test/e2e/templeton/deployers/config/webhcat/webhcat-site.xml index d8a5df1..8984503 100644 --- a/hcatalog/src/test/e2e/templeton/deployers/config/webhcat/webhcat-site.xml +++ b/hcatalog/src/test/e2e/templeton/deployers/config/webhcat/webhcat-site.xml @@ -78,10 +78,22 @@ HCat, Hive query, etc. + templeton.sqoop.archive + hdfs:///apps/templeton/sqoop-1.4.5.bin__hadoop-2.0.4-alpha.tar.gz + This should point to Sqoop tar that will be shipped to target node executing + the actual sqoop command. If not set, Sqoop is expected to be installed on every node of the + cluster. + + templeton.sqoop.path - ${env.SQOOP_HOME}/bin/sqoop + sqoop-1.4.5.bin__hadoop-2.0.4-alpha.tar.gz/sqoop-1.4.5.bin__hadoop-2.0.4-alpha/bin/sqoop The path to the Sqoop executable. + + templeton.sqoop.home + sqoop-1.4.5.bin__hadoop-2.0.4-alpha.tar.gz/sqoop-1.4.5.bin__hadoop-2.0.4-alpha + The path to the Sqoop home in the exploded archive. + templeton.controller.mr.child.opts diff --git a/hcatalog/src/test/e2e/templeton/deployers/deploy_e2e_artifacts.sh b/hcatalog/src/test/e2e/templeton/deployers/deploy_e2e_artifacts.sh index 29c3f3e..8a4621f 100755 --- a/hcatalog/src/test/e2e/templeton/deployers/deploy_e2e_artifacts.sh +++ b/hcatalog/src/test/e2e/templeton/deployers/deploy_e2e_artifacts.sh @@ -35,7 +35,6 @@ ${HADOOP_HOME}/bin/hdfs dfs -put ${HADOOP_HOME}/share/hadoop/mapreduce/hadoop-ma ${HADOOP_HOME}/bin/hdfs dfs -put ${HADOOP_HOME}/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-${HADOOP_VERSION}.jar webhcate2e/hclient.jar ${HADOOP_HOME}/bin/hdfs dfs -put ${HADOOP_HOME}/share/hadoop/tools/lib/hadoop-streaming-${HADOOP_VERSION}.jar /user/templeton/hadoop-streaming.jar - #must match config/webhcat/webhcat-stie.xml ${HADOOP_HOME}/bin/hdfs dfs -put ${PROJ_HOME}/packaging/target/apache-hive-${HIVE_VERSION}-bin.tar.gz /apps/templeton/apache-hive-${HIVE_VERSION}-bin.tar.gz # To run against Hadoop2 cluster, you have to build Pig tar yourself with @@ -47,7 +46,7 @@ ${HADOOP_HOME}/bin/hadoop fs -put ${PIG_PIGGYBANK_PATH} webhcate2e/ #${HADOOP_HOME}/bin/hadoop fs -put /Users/ekoifman/dev/data/jarsForTmplte2e/pig-0.12.0/contrib/piggybank/java/piggybank.jar webhcate2e/ -${HADOOP_HOME}/bin/hadoop fs -put ${HIVE_HOME}/lib/zookeeper-3.4.5.jar /apps/templeton/zookeeper-3.4.5.jar - +${HADOOP_HOME}/bin/hadoop fs -put /Users/ekoifman/dev/sqoop-1.4.5.bin__hadoop-2.0.4-alpha.tar.gz /apps/templeton/sqoop-1.4.5.bin__hadoop-2.0.4-alpha.tar.gz +${HADOOP_HOME}/bin/hadoop fs -put /Users/ekoifman/dev/mysql-connector-java-5.1.30/mysql-connector-java-5.1.30-bin.jar /apps/templeton/jdbc/mysql-connector-java.jar #check what got deployed -${HADOOP_HOME}/bin/hdfs dfs -ls /apps/templeton webhcate2e /user/templeton /user/hive/warehouse +${HADOOP_HOME}/bin/hdfs dfs -ls -R /apps/templeton webhcate2e /user/templeton /user/hive/warehouse diff --git a/hcatalog/src/test/e2e/templeton/deployers/env.sh b/hcatalog/src/test/e2e/templeton/deployers/env.sh index d66dd94..4a5f286 100755 --- a/hcatalog/src/test/e2e/templeton/deployers/env.sh +++ b/hcatalog/src/test/e2e/templeton/deployers/env.sh @@ -23,14 +23,13 @@ # define necessary env vars here and source it in other files export HADOOP_VERSION=2.4.1-SNAPSHOT -export HIVE_VERSION=0.14.0-SNAPSHOT +#export HIVE_VERSION=0.14.0-SNAPSHOT export PIG_VERSION=0.12.2-SNAPSHOT #Root of project source tree export PROJ_HOME=/Users/${USER}/dev/hive export HIVE_HOME=${PROJ_HOME}/packaging/target/apache-hive-${HIVE_VERSION}-bin/apache-hive-${HIVE_VERSION}-bin export HADOOP_HOME=/Users/${USER}/dev/hwxhadoop/hadoop-dist/target/hadoop-${HADOOP_VERSION} -export SQOOP_HOME=/Users/${USER}/dev/sqoop-1.4.4.bin__hadoop-2.0.4-alpha #Make sure Pig is built for the Hadoop version you are running export PIG_TAR_PATH=/Users/${USER}/dev/pig-${PIG_VERSION}-src/build diff --git a/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm b/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm index 23a472e..1c61bcc 100644 --- a/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm +++ b/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm @@ -193,6 +193,7 @@ sub globalSetup $globalHash->{'db_password'} = $ENV{'DB_PASSWORD'}; $globalHash->{'is_secure_mode'} = $ENV{'SECURE_MODE'}; + $globalHash->{'user_realm'} = $ENV{'USER_REALM'}; # add libexec location to the path if (defined($ENV{'PATH'})) { @@ -491,7 +492,14 @@ () } elsif(scalar @files > 1){ die "More than one keytab file found for user $user_name in $keytab_dir"; } - my @cmd = ('kinit', '-k', '-t', $files[0], $user_name); + my @cmd = (); + if (defined $testCmd->{'user_realm'}){ + my $user_name_with_realm_name = $user_name.'@'.$testCmd->{'user_realm'}; + @cmd = ('kinit', '-k', '-t', $files[0], $user_name_with_realm_name); + } + else{ + @cmd = ('kinit', '-k', '-t', $files[0], $user_name); + } print $log "Command @cmd"; IPC::Run::run(\@cmd, \undef, $log, $log) or die "Could not kinit as $user_name using " . $files[0] . " $ERRNO"; diff --git a/hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf b/hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf index 0fcc95c..f04ea31 100644 --- a/hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf +++ b/hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf @@ -524,7 +524,7 @@ $cfg = 'num' => 1, 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/sqoop?user.name=:UNAME:', - 'post_options' => ['command=export --connect :DB_CONNECTION_STRING: --username :DB_USER_NAME: --password :DB_PASSWORD: --export-dir :INPDIR_HDFS:/sqoop --table person','statusdir=sqoop.output' ], + 'post_options' => ['libdir=hdfs:///apps/templeton/jdbc', 'command=export --connect :DB_CONNECTION_STRING: --username :DB_USER_NAME: --password :DB_PASSWORD: --export-dir :INPDIR_HDFS:/sqoop --table person','statusdir=TestSqoop_:TNUM:' ], 'json_field_substr_match' => { 'id' => '\d+'}, #results 'status_code' => 200, @@ -539,7 +539,7 @@ $cfg = 'num' => 2, 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/sqoop?user.name=:UNAME:', - 'post_options' => ['files=:INPDIR_HDFS:/sqoopcommand.txt','command=import --connect :DB_CONNECTION_STRING: --username :DB_USER_NAME: --password :DB_PASSWORD: --options-file sqoopcommand.txt','statusdir=sqoop.output' ], + 'post_options' => ['libdir=hdfs:///apps/templeton/jdbc', 'files=:INPDIR_HDFS:/sqoopcommand.txt','command=import --connect :DB_CONNECTION_STRING: --username :DB_USER_NAME: --password :DB_PASSWORD: --options-file sqoopcommand.txt','statusdir=TestSqoop_:TNUM:' ], 'json_field_substr_match' => { 'id' => '\d+'}, #results 'status_code' => 200, @@ -554,7 +554,7 @@ $cfg = 'num' => 3, 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/sqoop?user.name=:UNAME:', - 'post_options' => ['files=:INPDIR_HDFS:/sqoopcommand.txt','statusdir=sqoop.output' ], + 'post_options' => ['files=:INPDIR_HDFS:/sqoopcommand.txt','statusdir=TestSqoop_:TNUM:' ], 'json_field_substr_match' => { 'error' => 'Must define Sqoop command or a optionsfile contains Sqoop command to run Sqoop job.'}, #results 'status_code' => 400, @@ -564,7 +564,7 @@ $cfg = 'num' => 4, 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/sqoop?user.name=:UNAME:', - 'post_options' => ['optionsfile=:INPDIR_HDFS:/sqoopcommand.txt','command=import --connect :DB_CONNECTION_STRING: --username :DB_USER_NAME: --password :DB_PASSWORD: --options-file sqoopcommand.txt','statusdir=sqoop.output' ], + 'post_options' => ['optionsfile=:INPDIR_HDFS:/sqoopcommand.txt','command=import --connect :DB_CONNECTION_STRING: --username :DB_USER_NAME: --password :DB_PASSWORD: --options-file sqoopcommand.txt','statusdir=TestSqoop_:TNUM:' ], 'json_field_substr_match' => { 'error' => 'Cannot set command and optionsfile at the same time.'}, #results 'status_code' => 400, diff --git a/hcatalog/streaming/pom.xml b/hcatalog/streaming/pom.xml index c5e1ae8..e653d39 100644 --- a/hcatalog/streaming/pom.xml +++ b/hcatalog/streaming/pom.xml @@ -20,7 +20,7 @@ org.apache.hive.hcatalog hive-hcatalog - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../pom.xml diff --git a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java index 903883d..a08f2f9 100644 --- a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java +++ b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java @@ -130,11 +130,16 @@ public StreamingConnection newConnection(final boolean createPartIfNotExists, Hi } /** - * Acquire a new connection to MetaStore for streaming + * Acquire a new connection to MetaStore for streaming. To connect using Kerberos, + * 'authenticatedUser' argument should have been used to do a kerberos login. Additionally the + * 'hive.metastore.kerberos.principal' setting should be set correctly either in hive-site.xml or + * in the 'conf' argument (if not null). If using hive-site.xml, it should be in classpath. + * * @param createPartIfNotExists If true, the partition specified in the endpoint * will be auto created if it does not exist + * @param conf HiveConf object to be used for the connection. Can be null. * @param authenticatedUser UserGroupInformation object obtained from successful authentication. - * Uses insecure mode if this argument is null. + * Uses non-secure mode if this argument is null. * @return * @throws ConnectionError if there is a connection problem * @throws InvalidPartition if specified partition is not valid (createPartIfNotExists = false) @@ -240,6 +245,7 @@ public String toString() { private final HiveEndPoint endPt; private final UserGroupInformation ugi; private final String username; + private final boolean secureMode; /** * @param endPoint end point to connect to @@ -261,7 +267,11 @@ private ConnectionImpl(HiveEndPoint endPoint, UserGroupInformation ugi, if (conf==null) { conf = HiveEndPoint.createHiveConf(this.getClass(), endPoint.metaStoreUri); } - this.msClient = getMetaStoreClient(endPoint, conf); + else { + overrideConfSettings(conf); + } + this.secureMode = ugi==null ? false : ugi.hasKerberosCredentials(); + this.msClient = getMetaStoreClient(endPoint, conf, secureMode); if (createPart && !endPoint.partitionVals.isEmpty()) { createPartitionIfNotExists(endPoint, msClient, conf); } @@ -425,13 +435,15 @@ private static String partSpecStr(List partKeys, ArrayList return buff.toString(); } - private static IMetaStoreClient getMetaStoreClient(HiveEndPoint endPoint, HiveConf conf) + private static IMetaStoreClient getMetaStoreClient(HiveEndPoint endPoint, HiveConf conf, boolean secureMode) throws ConnectionError { if (endPoint.metaStoreUri!= null) { conf.setVar(HiveConf.ConfVars.METASTOREURIS, endPoint.metaStoreUri); } - + if(secureMode) { + conf.setBoolVar(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL,true); + } try { return new HiveMetaStoreClient(conf); } catch (MetaException e) { @@ -828,14 +840,35 @@ private static LockRequest createLockRequest(final HiveEndPoint hiveEndPoint, static HiveConf createHiveConf(Class clazz, String metaStoreUri) { HiveConf conf = new HiveConf(clazz); - conf.setVar(HiveConf.ConfVars.HIVE_TXN_MANAGER, - "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager"); - conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, true); - conf.setBoolVar(HiveConf.ConfVars.METASTORE_EXECUTE_SET_UGI, true); if (metaStoreUri!= null) { - conf.setVar(HiveConf.ConfVars.METASTOREURIS, metaStoreUri); + setHiveConf(conf, HiveConf.ConfVars.METASTOREURIS, metaStoreUri); } + HiveEndPoint.overrideConfSettings(conf); return conf; } + private static void overrideConfSettings(HiveConf conf) { + setHiveConf(conf, HiveConf.ConfVars.HIVE_TXN_MANAGER, + "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager"); + setHiveConf(conf, HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, true); + setHiveConf(conf, HiveConf.ConfVars.METASTORE_EXECUTE_SET_UGI, true); + // Avoids creating Tez Client sessions internally as it takes much longer currently + setHiveConf(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "mr"); + } + + private static void setHiveConf(HiveConf conf, HiveConf.ConfVars var, String value) { + if( LOG.isDebugEnabled() ) { + LOG.debug("Overriding HiveConf setting : " + var + " = " + value); + } + conf.setVar(var, value); + } + + private static void setHiveConf(HiveConf conf, HiveConf.ConfVars var, boolean value) { + if( LOG.isDebugEnabled() ) { + LOG.debug("Overriding HiveConf setting : " + var + " = " + value); + } + conf.setBoolVar(var, value); + } + + } // class HiveEndPoint diff --git a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java index ec99728..6d332be 100644 --- a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java +++ b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java @@ -731,7 +731,7 @@ public void createDbAndTable(IMetaStoreClient client, String databaseName, throws Exception { Database db = new Database(); db.setName(databaseName); - String dbLocation = "raw://" + dbFolder.newFolder(databaseName + ".db").getCanonicalPath(); + String dbLocation = "raw://" + dbFolder.newFolder(databaseName + ".db").toURI().getPath(); db.setLocationUri(dbLocation); client.createDatabase(db); diff --git a/hcatalog/webhcat/java-client/pom.xml b/hcatalog/webhcat/java-client/pom.xml index 0511cad..ebef9f1 100644 --- a/hcatalog/webhcat/java-client/pom.xml +++ b/hcatalog/webhcat/java-client/pom.xml @@ -25,7 +25,7 @@ org.apache.hive.hcatalog hive-hcatalog - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../../pom.xml diff --git a/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java b/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java index b3afa72..41cb44e 100644 --- a/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java +++ b/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java @@ -329,8 +329,7 @@ public void renameTable(String dbName, String oldName, String newName) throws HCatException { List hcatPtns = new ArrayList(); try { - Table table = hmsClient.getTable(dbName, tblName); - HCatTable hcatTable = new HCatTable(table); + HCatTable hcatTable = getTable(dbName, tblName); List hivePtns = hmsClient.listPartitions( checkDB(dbName), tblName, (short) -1); for (Partition ptn : hivePtns) { @@ -374,7 +373,7 @@ public HCatPartition getPartition(String dbName, String tableName, Map partitionSpec) throws HCatException { HCatPartition partition = null; try { - HCatTable hcatTable = getTable(checkDB(dbName), tableName); + HCatTable hcatTable = getTable(dbName, tableName); List partitionColumns = hcatTable.getPartCols(); if (partitionColumns.size() != partitionSpec.size()) { throw new HCatException("Partition-spec doesn't have the right number of partition keys."); @@ -494,7 +493,7 @@ private void dropPartition(Partition partition, boolean ifExists) try { HCatTable table = getTable(dbName, tblName); List hivePtns = hmsClient.listPartitionsByFilter( - checkDB(dbName), tblName, filter, (short) -1); + table.getDbName(), table.getTableName(), filter, (short) -1); for (Partition ptn : hivePtns) { hcatPtns.add(new HCatPartition(table, ptn)); } diff --git a/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatPartition.java b/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatPartition.java index 52586a0..0d714ff 100644 --- a/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatPartition.java +++ b/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatPartition.java @@ -188,6 +188,15 @@ public String getDatabaseName() { } /** + * Gets the partition columns of the table. + * + * @return the partition columns + */ + public List getPartColumns() { + return hcatTable.getPartCols(); + } + + /** * Gets the input format. * * @return the input format diff --git a/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java b/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java index f9f7b04..bc90ffe 100644 --- a/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java +++ b/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java @@ -157,7 +157,7 @@ public void testBasicDDLCommands() throws Exception { assertTrue(testDb.getProperties().size() == 0); String warehouseDir = System .getProperty("test.warehouse.dir", "/user/hive/warehouse"); - String expectedDir = fixPath(warehouseDir).replaceFirst("pfile:///", "pfile:/"); + String expectedDir = warehouseDir.replaceFirst("pfile:///", "pfile:/"); assertEquals(expectedDir + "/" + db + ".db", testDb.getLocation()); ArrayList cols = new ArrayList(); cols.add(new HCatFieldSchema("id", Type.INT, "id comment")); @@ -758,6 +758,11 @@ public void testDropPartitionsWithPartialSpec() throws Exception { assertEquals("Unexpected number of partitions.", 1, partitions.size()); assertArrayEquals("Mismatched partition.", new String[]{"2011_12_31", "AB"}, partitions.get(0).getValues().toArray()); + List partColumns = partitions.get(0).getPartColumns(); + assertEquals(2, partColumns.size()); + assertEquals("dt", partColumns.get(0).getName()); + assertEquals("grid", partColumns.get(1).getName()); + client.dropDatabase(dbName, false, HCatClient.DropDBMode.CASCADE); } catch (Exception unexpected) { diff --git a/hcatalog/webhcat/svr/.gitignore b/hcatalog/webhcat/svr/.gitignore new file mode 100644 index 0000000..916e17c --- /dev/null +++ b/hcatalog/webhcat/svr/.gitignore @@ -0,0 +1 @@ +dependency-reduced-pom.xml diff --git a/hcatalog/webhcat/svr/pom.xml b/hcatalog/webhcat/svr/pom.xml index 8c94717..4969708 100644 --- a/hcatalog/webhcat/svr/pom.xml +++ b/hcatalog/webhcat/svr/pom.xml @@ -25,7 +25,7 @@ org.apache.hive.hcatalog hive-hcatalog - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../../pom.xml @@ -38,7 +38,7 @@ - + org.apache.hive.hcatalog @@ -73,6 +73,11 @@ ${commons-exec.version} + org.apache.curator + curator-framework + ${curator.version} + + org.apache.zookeeper zookeeper ${zookeeper.version} diff --git a/hcatalog/webhcat/svr/src/main/bin/webhcat_server.sh b/hcatalog/webhcat/svr/src/main/bin/webhcat_server.sh index f0ce3a3..0be8dde 100644 --- a/hcatalog/webhcat/svr/src/main/bin/webhcat_server.sh +++ b/hcatalog/webhcat/svr/src/main/bin/webhcat_server.sh @@ -138,6 +138,9 @@ function start_webhcat() { log "starting ..." log "$start_cmd" + if [ ! -d ${WEBHCAT_LOG_DIR} ]; then + mkdir ${WEBHCAT_LOG_DIR} + fi nohup $start_cmd >>$CONSOLE_LOG 2>>$ERROR_LOG & local pid=$! diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java index 0342080..77952be 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java @@ -119,6 +119,7 @@ public static final String HIVE_PROPS_NAME = "templeton.hive.properties"; public static final String SQOOP_ARCHIVE_NAME = "templeton.sqoop.archive"; public static final String SQOOP_PATH_NAME = "templeton.sqoop.path"; + public static final String SQOOP_HOME_PATH = "templeton.sqoop.home"; public static final String LIB_JARS_NAME = "templeton.libjars"; public static final String PIG_ARCHIVE_NAME = "templeton.pig.archive"; public static final String PIG_PATH_NAME = "templeton.pig.path"; @@ -297,6 +298,7 @@ private boolean loadOneClasspathConfig(String fname) { public String hiveArchive() { return get(HIVE_ARCHIVE_NAME); } public String sqoopPath() { return get(SQOOP_PATH_NAME); } public String sqoopArchive() { return get(SQOOP_ARCHIVE_NAME); } + public String sqoopHome() { return get(SQOOP_HOME_PATH); } public String streamingJar() { return get(STREAMING_JAR_NAME); } public String kerberosSecret() { return get(KERBEROS_SECRET); } public String kerberosPrincipal(){ return get(KERBEROS_PRINCIPAL); } diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java index ebe1179..e5832a8 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java @@ -85,6 +85,9 @@ public EnqueueBean run(String user, Map userArgs, String jar, St if (TempletonUtils.isset(libjars)) { String libjarsListAsString = TempletonUtils.hadoopFsListAsString(libjars, appConf, runAs); + //This will work only if the files are local files on webhcat server + // (which is not very useful since users might not have access to that file system). + //This is likely the HIVE-5188 issue args.add("-libjars"); args.add(TempletonUtils.quoteForWindows(libjarsListAsString)); } diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java index c7e92cf..71a3107 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java @@ -121,6 +121,8 @@ public String run() throws Exception { JobType jobType) { ArrayList args = new ArrayList(); + //note that in ToolRunner this is expected to be a local FS path + //see GenericOptionsParser.getLibJars() args.add("-libjars"); // Include shim and admin specified libjars @@ -136,6 +138,7 @@ public String run() throws Exception { // Internal vars addDef(args, TempletonControllerJob.STATUSDIR_NAME, statusdir); + //Use of ToolRunner "-files" option could be considered here addDef(args, TempletonControllerJob.COPY_NAME, TempletonUtils.encodeArray(copyFiles)); addDef(args, TempletonControllerJob.OVERRIDE_CLASSPATH, diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java index 9b18a4c..a28e04d 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java @@ -778,6 +778,7 @@ public EnqueueBean pig(@FormParam("execute") String execute, * @param optionsFile name of option file which contains Sqoop command to run * @param otherFiles additional files to be shipped to the launcher, such as option files which contain part of the Sqoop command + * @param libdir dir containing JDBC jars that Sqoop will need to interact with the database * @param statusdir where the stderr/stdout of templeton controller job goes * @param callback URL which WebHCat will call when the sqoop job finishes * @param enablelog whether to collect mapreduce log into statusdir/logs @@ -787,12 +788,13 @@ public EnqueueBean pig(@FormParam("execute") String execute, @Produces({MediaType.APPLICATION_JSON}) public EnqueueBean sqoop(@FormParam("command") String command, @FormParam("optionsfile") String optionsFile, + @FormParam("libdir") String libdir, @FormParam("files") String otherFiles, @FormParam("statusdir") String statusdir, @FormParam("callback") String callback, @FormParam("enablelog") boolean enablelog) throws NotAuthorizedException, BusyException, BadParam, QueueException, - ExecuteException, IOException, InterruptedException { + IOException, InterruptedException { verifyUser(); if (command == null && optionsFile == null) throw new BadParam("Must define Sqoop command or a optionsfile contains Sqoop command to run Sqoop job."); @@ -805,13 +807,14 @@ public EnqueueBean sqoop(@FormParam("command") String command, userArgs.put("user.name", getDoAsUser()); userArgs.put("command", command); userArgs.put("optionsfile", optionsFile); + userArgs.put("libdir", libdir); userArgs.put("files", otherFiles); userArgs.put("statusdir", statusdir); userArgs.put("callback", callback); userArgs.put("enablelog", Boolean.toString(enablelog)); SqoopDelegator d = new SqoopDelegator(appConf); return d.run(getDoAsUser(), userArgs, command, optionsFile, otherFiles, - statusdir, callback, getCompletedUrl(), enablelog); + statusdir, callback, getCompletedUrl(), enablelog, libdir); } /** diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java index 24b45ef..292f695 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java @@ -25,7 +25,10 @@ import java.util.List; import java.util.Map; -import org.apache.commons.exec.ExecuteException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.Path; +import org.apache.hive.hcatalog.templeton.tool.JobSubmissionConstants; import org.apache.hive.hcatalog.templeton.tool.TempletonControllerJob; import org.apache.hive.hcatalog.templeton.tool.TempletonUtils; @@ -35,6 +38,7 @@ * This is the backend of the Sqoop web service. */ public class SqoopDelegator extends LauncherDelegator { + private static final Log LOG = LogFactory.getLog(SqoopDelegator.class); public SqoopDelegator(AppConfig appConf) { super(appConf); @@ -43,24 +47,29 @@ public SqoopDelegator(AppConfig appConf) { public EnqueueBean run(String user, Map userArgs, String command, String optionsFile, String otherFiles, String statusdir, - String callback, String completedUrl, boolean enablelog) + String callback, String completedUrl, boolean enablelog, String libdir) throws NotAuthorizedException, BadParam, BusyException, QueueException, - ExecuteException, IOException, InterruptedException + IOException, InterruptedException { + if(TempletonUtils.isset(appConf.sqoopArchive())) { + if(!TempletonUtils.isset(appConf.sqoopPath()) && !TempletonUtils.isset(appConf.sqoopHome())) { + throw new IllegalStateException("If '" + AppConfig.SQOOP_ARCHIVE_NAME + "' is defined, '" + + AppConfig.SQOOP_PATH_NAME + "' and '" + AppConfig.SQOOP_HOME_PATH + "' must be defined"); + } + } runAs = user; List args = makeArgs(command, optionsFile, otherFiles, statusdir, - completedUrl, enablelog); + completedUrl, enablelog, libdir); return enqueueController(user, userArgs, callback, args); } - - List makeArgs(String command, String optionsFile, String otherFiles, - String statusdir, String completedUrl, boolean enablelog) + private List makeArgs(String command, String optionsFile, String otherFiles, + String statusdir, String completedUrl, boolean enablelog, String libdir) throws BadParam, IOException, InterruptedException { ArrayList args = new ArrayList(); try { - args.addAll(makeBasicArgs(optionsFile, otherFiles, statusdir, completedUrl, enablelog)); + args.addAll(makeBasicArgs(optionsFile, otherFiles, statusdir, completedUrl, enablelog, libdir)); args.add("--"); TempletonUtils.addCmdForWindows(args); args.add(appConf.sqoopPath()); @@ -89,7 +98,7 @@ public EnqueueBean run(String user, } private List makeBasicArgs(String optionsFile, String otherFiles, - String statusdir, String completedUrl, boolean enablelog) + String statusdir, String completedUrl, boolean enablelog, String libdir) throws URISyntaxException, FileNotFoundException, IOException, InterruptedException { @@ -101,9 +110,34 @@ public EnqueueBean run(String user, String[] ofs = TempletonUtils.hadoopFsListAsArray(otherFiles, appConf, runAs); allFiles.addAll(Arrays.asList(ofs)); } + if(TempletonUtils.isset(libdir) && TempletonUtils.isset(appConf.sqoopArchive())) { + /**Sqoop accesses databases via JDBC. This means it needs to have appropriate JDBC + drivers available. Normally, the user would install Sqoop and place these jars + into SQOOP_HOME/lib. When WebHCat is configured to auto-ship the Sqoop tar file, we + need to make sure that relevant JDBC jars are available on target node. + The user is expected to place any JDBC jars into an HDFS directory and specify this + dir in "libdir" parameter. All the files in this dir will be copied to lib/ of the + exploded Sqoop tar ball on target node. + {@link org.apache.hive.hcatalog.templeton.tool.LaunchMapper#handleSqoop(org.apache.hadoop.conf.Configuration, java.util.Map)} + */ + LOG.debug("libdir=" + libdir); + List jarList = TempletonUtils.hadoopFsListChildren(libdir, appConf, runAs); + if(TempletonUtils.isset(jarList)) { + StringBuilder sb = new StringBuilder(); + for(Path jar : jarList) { + allFiles.add(jar.toString()); + sb.append(jar.getName()).append(','); + } + sb.setLength(sb.length() - 1); + //we use the same mechanism to copy "files"/"otherFiles" and "libdir", but we only want to put + //contents of "libdir" in Sqoop/lib, thus we pass the list of names here + addDef(args, JobSubmissionConstants.Sqoop.LIB_JARS, sb.toString()); + addDef(args, AppConfig.SQOOP_HOME_PATH, appConf.get(AppConfig.SQOOP_HOME_PATH)); + } + } args.addAll(makeLauncherArgs(appConf, statusdir, completedUrl, allFiles, enablelog, JobType.SQOOP)); - if (appConf.sqoopArchive() != null && !appConf.sqoopArchive().equals("")) { + if(TempletonUtils.isset(appConf.sqoopArchive())) { args.add("-archives"); args.add(appConf.sqoopArchive()); } diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/HDFSStorage.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/HDFSStorage.java index fb46b58..9c73a73 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/HDFSStorage.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/HDFSStorage.java @@ -25,9 +25,7 @@ import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -116,32 +114,6 @@ public String getField(Type type, String id, String key) { } @Override - public Map getFields(Type type, String id) { - HashMap map = new HashMap(); - BufferedReader in = null; - Path p = new Path(getPath(type) + "/" + id); - try { - for (FileStatus status : fs.listStatus(p)) { - in = new BufferedReader(new InputStreamReader(fs.open(status.getPath()))); - String line = null; - String val = ""; - while ((line = in.readLine()) != null) { - if (!val.equals("")) { - val += "\n"; - } - val += line; - } - map.put(status.getPath().getName(), val); - } - } catch (IOException e) { - LOG.trace("Couldn't find " + p); - } finally { - close(in); - } - return map; - } - - @Override public boolean delete(Type type, String id) throws NotFoundException { Path p = new Path(getPath(type) + "/" + id); try { @@ -153,14 +125,6 @@ public boolean delete(Type type, String id) throws NotFoundException { return false; } - @Override - public List getAll() { - ArrayList allNodes = new ArrayList(); - for (Type type : Type.values()) { - allNodes.addAll(getAllForType(type)); - } - return allNodes; - } @Override public List getAllForType(Type type) { @@ -177,40 +141,6 @@ public boolean delete(Type type, String id) throws NotFoundException { } @Override - public List getAllForKey(String key, String value) { - ArrayList allNodes = new ArrayList(); - try { - for (Type type : Type.values()) { - allNodes.addAll(getAllForTypeAndKey(type, key, value)); - } - } catch (Exception e) { - LOG.trace("Couldn't find children for key " + key + ": " + - e.getMessage()); - } - return allNodes; - } - - @Override - public List getAllForTypeAndKey(Type type, String key, String value) { - ArrayList allNodes = new ArrayList(); - HashMap map = new HashMap(); - try { - for (FileStatus status : - fs.listStatus(new Path(getPath(type)))) { - map = (HashMap) - getFields(type, status.getPath().getName()); - if (map.get(key).equals(value)) { - allNodes.add(status.getPath().getName()); - } - } - } catch (Exception e) { - LOG.trace("Couldn't find children for key " + key + ": " + - e.getMessage()); - } - return allNodes; - } - - @Override public void openStorage(Configuration config) throws IOException { storage_root = config.get(TempletonStorage.STORAGE_ROOT); if (fs == null) { diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobStateTracker.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobStateTracker.java index da3ad20..41fd82f 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobStateTracker.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobStateTracker.java @@ -24,19 +24,30 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.curator.framework.CuratorFramework; import org.apache.hadoop.conf.Configuration; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.ZooKeeper; import org.apache.zookeeper.ZooDefs.Ids; -import org.apache.zookeeper.data.Stat; +/* + * The general idea here is to create + * /created/1 + * /created/2 + * /created/3 .... + * for each job submitted. The node number is generated by ZK (PERSISTENT_SEQUENTIAL) and the + * payload is the JobId. Basically this keeps track of the order in which jobs were submitted, + * and ZooKeeperCleanup uses this to purge old job info. + * Since the /jobs/ node has a create/update timestamp + * (http://zookeeper.apache.org/doc/trunk/zookeeperProgrammers.html#sc_zkStatStructure) this whole + * thing can be removed. +*/ public class JobStateTracker { // The path to the tracking root private String job_trackingroot = null; // The zookeeper connection to use - private ZooKeeper zk; + private CuratorFramework zk; // The id of the tracking node -- must be a SEQUENTIAL node private String trackingnode; @@ -51,7 +62,7 @@ * Constructor for a new node -- takes the jobid of an existing job * */ - public JobStateTracker(String node, ZooKeeper zk, boolean nodeIsTracker, + public JobStateTracker(String node, CuratorFramework zk, boolean nodeIsTracker, String job_trackingpath) { this.zk = zk; if (nodeIsTracker) { @@ -65,30 +76,25 @@ public JobStateTracker(String node, ZooKeeper zk, boolean nodeIsTracker, /** * Create the parent znode for this job state. */ - public void create() - throws IOException { - String[] paths = ZooKeeperStorage.getPaths(job_trackingroot); - for (String znode : paths) { - try { - zk.create(znode, new byte[0], - Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); - } catch (KeeperException.NodeExistsException e) { - } catch (Exception e) { - throw new IOException("Unable to create parent nodes"); - } + public void create() throws IOException { + try { + zk.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT) + .withACL(Ids.OPEN_ACL_UNSAFE).forPath(job_trackingroot); + } catch (KeeperException.NodeExistsException e) { + //root must exist already + } catch (Exception e) { + throw new IOException("Unable to create parent nodes"); } try { - trackingnode = zk.create(makeTrackingZnode(), jobid.getBytes(), - Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT_SEQUENTIAL); + trackingnode = zk.create().withMode(CreateMode.PERSISTENT_SEQUENTIAL) + .withACL(Ids.OPEN_ACL_UNSAFE).forPath(makeTrackingZnode(), jobid.getBytes()); } catch (Exception e) { throw new IOException("Unable to create " + makeTrackingZnode()); } } - - public void delete() - throws IOException { + public void delete() throws IOException { try { - zk.delete(makeTrackingJobZnode(trackingnode), -1); + zk.delete().forPath(makeTrackingJobZnode(trackingnode)); } catch (Exception e) { // Might have been deleted already LOG.info("Couldn't delete " + makeTrackingJobZnode(trackingnode)); @@ -101,13 +107,10 @@ public void delete() */ public String getJobID() throws IOException { try { - return new String(zk.getData(makeTrackingJobZnode(trackingnode), - false, new Stat())); - } catch (KeeperException e) { + return new String(zk.getData().forPath(makeTrackingJobZnode(trackingnode))); + } catch (Exception e) { // It was deleted during the transaction - throw new IOException("Node already deleted " + trackingnode); - } catch (InterruptedException e) { - throw new IOException("Couldn't read node " + trackingnode); + throw new IOException("Node already deleted " + trackingnode, e); } } @@ -129,13 +132,13 @@ public String makeTrackingJobZnode(String nodename) { * Get the list of tracking jobs. These can be used to determine which jobs have * expired. */ - public static List getTrackingJobs(Configuration conf, ZooKeeper zk) + public static List getTrackingJobs(Configuration conf, CuratorFramework zk) throws IOException { ArrayList jobs = new ArrayList(); try { - for (String myid : zk.getChildren( + for (String myid : zk.getChildren().forPath( conf.get(TempletonStorage.STORAGE_ROOT) - + ZooKeeperStorage.TRACKINGDIR, false)) { + + ZooKeeperStorage.TRACKINGDIR)) { jobs.add(myid); } } catch (Exception e) { diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java index ccf6107..1252105 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java @@ -58,4 +58,10 @@ public static final String HCAT_HOME = "HCAT_HOME"; public static final String PIG_OPTS = "PIG_OPTS"; } + public static interface Sqoop { + /** + * comma-separated list of jar names (short name) which are needed for Sqoop JDBC access + */ + public static final String LIB_JARS = "templeton.sqoop.lib.jar"; + } } diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java index fb9d767..557d1da 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java @@ -18,6 +18,7 @@ */ package org.apache.hive.hcatalog.templeton.tool; +import com.google.common.io.Files; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -33,6 +34,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.StringUtils; +import org.apache.hive.hcatalog.templeton.AppConfig; import org.apache.hive.hcatalog.templeton.BadParam; import org.apache.hive.hcatalog.templeton.LauncherDelegator; @@ -89,6 +91,24 @@ private static void handlePigEnvVars(Configuration conf, Map env env.put(PigConstants.PIG_OPTS, pigOpts.toString()); } } + + /** + * {@link #copyLocal(String, org.apache.hadoop.conf.Configuration)} should be called before this + * See {@link org.apache.hive.hcatalog.templeton.SqoopDelegator#makeBasicArgs(String, String, String, String, boolean, String)} + * for more comments + */ + private static void handleSqoop(Configuration conf, Map env) throws IOException { + if(TempletonUtils.isset(conf.get(Sqoop.LIB_JARS))) { + //LIB_JARS should only be set if Sqoop is auto-shipped + LOG.debug(Sqoop.LIB_JARS + "=" + conf.get(Sqoop.LIB_JARS)); + //copy these (which have now been localized) jars to sqoop/lib + String destDir = conf.get(AppConfig.SQOOP_HOME_PATH) + File.separator + "lib"; + String[] files = conf.getStrings(Sqoop.LIB_JARS); + for(String f : files) { + Files.copy(new File(f), new File(destDir + File.separator + f)); + } + } + } protected Process startJob(Context context, String user, String overrideClasspath) throws IOException, InterruptedException { Configuration conf = context.getConfiguration(); @@ -108,6 +128,7 @@ protected Process startJob(Context context, String user, String overrideClasspat removeEnv.add("mapredcommand"); Map env = TempletonUtils.hadoopUserEnv(user, overrideClasspath); handlePigEnvVars(conf, env); + handleSqoop(conf, env); List jarArgsList = new LinkedList(Arrays.asList(jarArgs)); handleTokenFile(jarArgsList, JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER, "mapreduce.job.credentials.binary"); handleTokenFile(jarArgsList, JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER_TEZ, "tez.credentials.path"); diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonStorage.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonStorage.java index 97d572c..cf27ffa 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonStorage.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonStorage.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.util.List; -import java.util.Map; import org.apache.hadoop.conf.Configuration; @@ -45,7 +44,7 @@ public interface TempletonStorage { // These are the possible types referenced by 'type' below. public enum Type { - UNKNOWN, JOB, JOBTRACKING, TEMPLETONOVERHEAD + UNKNOWN, JOB, JOBTRACKING } public static final String STORAGE_CLASS = "templeton.storage.class"; @@ -79,20 +78,6 @@ public void saveField(Type type, String id, String key, String val) public String getField(Type type, String id, String key); /** - * Get all the name/value pairs stored for this id. - * Be careful using getFields() -- optimistic locking will mean that - * your odds of a conflict are decreased if you read/write one field - * at a time. getFields() is intended for read-only usage. - * - * If the type is UNKNOWN, search for the id in all types. - * - * @param type The data type (as listed above) - * @param id The String id of this data grouping (jobid, etc.) - * @return A Map of key/value pairs found for this type/id. - */ - public Map getFields(Type type, String id); - - /** * Delete a data grouping (all data for a jobid, all tracking data * for a job, etc.). If the type is UNKNOWN, search for the id * in all types. @@ -105,13 +90,6 @@ public void saveField(Type type, String id, String key, String val) public boolean delete(Type type, String id) throws NotFoundException; /** - * Get the id of each data grouping in the storage system. - * - * @return An ArrayList of ids. - */ - public List getAll(); - - /** * Get the id of each data grouping of a given type in the storage * system. * @param type The data type (as listed above) @@ -120,26 +98,6 @@ public void saveField(Type type, String id, String key, String val) public List getAllForType(Type type); /** - * Get the id of each data grouping that has the specific key/value - * pair. - * @param key The name of the field to search for - * @param value The value of the field to search for - * @return An ArrayList of ids. - */ - public List getAllForKey(String key, String value); - - /** - * Get the id of each data grouping of a given type that has the - * specific key/value pair. - * @param type The data type (as listed above) - * @param key The name of the field to search for - * @param value The value of the field to search for - * @return An ArrayList of ids. - */ - public List getAllForTypeAndKey(Type type, String key, - String value); - - /** * For storage methods that require a connection, this is a hint * that it's time to open a connection. */ diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java index d2dec54..902d786 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java @@ -29,6 +29,7 @@ import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.Enumeration; import java.util.HashMap; import java.util.List; @@ -39,8 +40,10 @@ import javax.ws.rs.core.UriBuilder; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.StringUtils; @@ -211,6 +214,28 @@ public static String hadoopFsFilename(String fname, Configuration conf, String u } /** + * Returns all files (non-recursive) in {@code dirName} + */ + public static List hadoopFsListChildren(String dirName, Configuration conf, String user) + throws URISyntaxException, IOException, InterruptedException { + + Path p = hadoopFsPath(dirName, conf, user); + FileSystem fs = p.getFileSystem(conf); + if(!fs.exists(p)) { + return Collections.emptyList(); + } + List children = ShimLoader.getHadoopShims().listLocatedStatus(fs, p, null); + if(!isset(children)) { + return Collections.emptyList(); + } + List files = new ArrayList(); + for(FileStatus stat : children) { + files.add(stat.getPath()); + } + return files; + } + + /** * @return true iff we are sure the file is not there. */ public static boolean hadoopFsIsMissing(FileSystem fs, Path p) { @@ -239,8 +264,7 @@ public static String addUserHomeDirectoryIfApplicable(String origPathStr, String } public static Path hadoopFsPath(String fname, final Configuration conf, String user) - throws URISyntaxException, IOException, - InterruptedException { + throws URISyntaxException, IOException, InterruptedException { if (fname == null || conf == null) { return null; } diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/ZooKeeperCleanup.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/ZooKeeperCleanup.java index 4f5e2d9..24336e2 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/ZooKeeperCleanup.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/ZooKeeperCleanup.java @@ -24,8 +24,8 @@ import java.util.List; import java.util.Date; +import org.apache.curator.framework.CuratorFramework; import org.apache.hadoop.conf.Configuration; -import org.apache.zookeeper.ZooKeeper; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -89,7 +89,7 @@ public static void startInstance(Configuration appConf) throws IOException { * @throws IOException */ public void run() { - ZooKeeper zk = null; + CuratorFramework zk = null; List nodes = null; isRunning = true; while (!stop) { @@ -112,13 +112,7 @@ public void run() { } catch (Exception e) { LOG.error("Cleanup cycle failed: " + e.getMessage()); } finally { - if (zk != null) { - try { - zk.close(); - } catch (InterruptedException e) { - // We're trying to exit anyway, just ignore. - } - } + if (zk != null) zk.close(); } long sleepMillis = (long) (Math.random() * interval); @@ -140,7 +134,7 @@ public void run() { * * @throws IOException */ - public List getChildList(ZooKeeper zk) { + public List getChildList(CuratorFramework zk) { try { List jobs = JobStateTracker.getTrackingJobs(appConf, zk); Collections.sort(jobs); @@ -154,7 +148,7 @@ public void run() { /** * Check to see if a job is more than maxage old, and delete it if so. */ - public boolean checkAndDelete(String node, ZooKeeper zk) { + public boolean checkAndDelete(String node, CuratorFramework zk) { JobState state = null; try { JobStateTracker tracker = new JobStateTracker(node, zk, true, @@ -167,8 +161,11 @@ public boolean checkAndDelete(String node, ZooKeeper zk) { // an error in creation, and we want to delete it anyway. long then = 0; if (state.getCreated() != null) { + //this is set in ZooKeeperStorage.create() then = state.getCreated(); } + //todo: this should check that the job actually completed and likely use completion time + //which is not tracked directly but available on /jobs/ node via "mtime" in Stat if (now - then > maxage) { LOG.info("Deleting " + tracker.getJobID()); state.delete(); @@ -177,7 +174,7 @@ public boolean checkAndDelete(String node, ZooKeeper zk) { } return false; } catch (Exception e) { - LOG.info("checkAndDelete failed for " + node); + LOG.info("checkAndDelete failed for " + node + " due to: " + e.getMessage()); // We don't throw a new exception for this -- just keep going with the // next one. return true; diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/ZooKeeperStorage.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/ZooKeeperStorage.java index ecde598..b9fc4b1 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/ZooKeeperStorage.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/ZooKeeperStorage.java @@ -19,21 +19,18 @@ package org.apache.hive.hcatalog.templeton.tool; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.hadoop.conf.Configuration; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.Watcher; import org.apache.zookeeper.ZooDefs.Ids; -import org.apache.zookeeper.ZooKeeper; /** * A storage implementation based on storing everything in ZooKeeper. @@ -60,29 +57,29 @@ private static final Log LOG = LogFactory.getLog(ZooKeeperStorage.class); - private ZooKeeper zk; + private CuratorFramework zk; /** * Open a ZooKeeper connection for the JobState. */ - public static ZooKeeper zkOpen(String zkHosts, int zkSessionTimeout) + public static CuratorFramework zkOpen(String zkHosts, int zkSessionTimeoutMs) throws IOException { - return new ZooKeeper(zkHosts, - zkSessionTimeout, - new Watcher() { - @Override - synchronized public void process(WatchedEvent event) { - } - }); + //do we need to add a connection status listener? What will that do? + ExponentialBackoffRetry retryPolicy = new ExponentialBackoffRetry(1000, 3); + CuratorFramework zk = CuratorFrameworkFactory.newClient(zkHosts, zkSessionTimeoutMs, + CuratorFrameworkFactory.builder().getConnectionTimeoutMs(), retryPolicy); + zk.start(); + return zk; } /** * Open a ZooKeeper connection for the JobState. */ - public static ZooKeeper zkOpen(Configuration conf) - throws IOException { + public static CuratorFramework zkOpen(Configuration conf) throws IOException { + /*the silly looking call to Builder below is to get the default value of session timeout + from Curator which itself exposes it as system property*/ return zkOpen(conf.get(ZK_HOSTS), - conf.getInt(ZK_SESSION_TIMEOUT, 30000)); + conf.getInt(ZK_SESSION_TIMEOUT, CuratorFrameworkFactory.builder().getSessionTimeoutMs())); } public ZooKeeperStorage() { @@ -93,15 +90,9 @@ public ZooKeeperStorage() { /** * Close this ZK connection. */ - public void close() - throws IOException { + public void close() throws IOException { if (zk != null) { - try { - zk.close(); - zk = null; - } catch (InterruptedException e) { - throw new IOException("Closing ZooKeeper connection", e); - } + zk.close(); } } @@ -118,48 +109,54 @@ public void startCleanup(Configuration config) { */ public void create(Type type, String id) throws IOException { + boolean wasCreated = false; try { - String[] paths = getPaths(makeZnode(type, id)); - boolean wasCreated = false; - for (String znode : paths) { - try { - zk.create(znode, new byte[0], - Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); - wasCreated = true; - } catch (KeeperException.NodeExistsException e) { + zk.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT).withACL(Ids.OPEN_ACL_UNSAFE).forPath(makeZnode(type, id)); + wasCreated = true; + } + catch(KeeperException.NodeExistsException ex) { + //we just created top level node for this jobId + } + catch(Exception ex) { + throw new IOException("Error creating " + makeZnode(type, id), ex); + } + if(wasCreated) { + try { + // Really not sure if this should go here. Will have + // to see how the storage mechanism evolves. + if (type.equals(Type.JOB)) { + JobStateTracker jt = new JobStateTracker(id, zk, false, job_trackingpath); + jt.create(); } - } - if (wasCreated) { + } catch (Exception e) { + LOG.error("Error tracking (jobId=" + id + "): " + e.getMessage()); + // If we couldn't create the tracker node, don't create the main node. try { - // Really not sure if this should go here. Will have - // to see how the storage mechanism evolves. - if (type.equals(Type.JOB)) { - JobStateTracker jt = new JobStateTracker(id, zk, false, - job_trackingpath); - jt.create(); - } - } catch (Exception e) { - LOG.warn("Error tracking: " + e.getMessage()); - // If we couldn't create the tracker node, don't - // create the main node. - zk.delete(makeZnode(type, id), -1); + zk.delete().forPath(makeZnode(type, id));//default version is -1 + } + catch(Exception ex) { + //EK: it's not obvious that this is the right logic, if we don't record the 'callback' + //for example and never notify the client of job completion + throw new IOException("Failed to delete " + makeZnode(type, id) + ":" + ex); } } - if (zk.exists(makeZnode(type, id), false) == null) + } + try { + if (zk.checkExists().forPath(makeZnode(type, id)) == null) { throw new IOException("Unable to create " + makeZnode(type, id)); - if (wasCreated) { - try { - saveField(type, id, "created", - Long.toString(System.currentTimeMillis())); - } catch (NotFoundException nfe) { - // Wow, something's really wrong. - throw new IOException("Couldn't write to node " + id, nfe); - } } - } catch (KeeperException e) { - throw new IOException("Creating " + id, e); - } catch (InterruptedException e) { - throw new IOException("Creating " + id, e); + } + catch (Exception ex) { + throw new IOException(ex); + } + if (wasCreated) { + try { + saveField(type, id, "created", + Long.toString(System.currentTimeMillis())); + } catch (NotFoundException nfe) { + // Wow, something's really wrong. + throw new IOException("Couldn't write to node " + id, nfe); + } } } @@ -198,25 +195,14 @@ public String getPath(Type type) { /** * A helper method that sets a field value. - * @param type - * @param id - * @param name - * @param val - * @throws KeeperException - * @throws UnsupportedEncodingException - * @throws InterruptedException + * @throws java.lang.Exception */ - private void setFieldData(Type type, String id, String name, String val) - throws KeeperException, UnsupportedEncodingException, InterruptedException { + private void setFieldData(Type type, String id, String name, String val) throws Exception { try { - zk.create(makeFieldZnode(type, id, name), - val.getBytes(ENCODING), - Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); + zk.create().withMode(CreateMode.PERSISTENT).withACL(Ids.OPEN_ACL_UNSAFE) + .forPath(makeFieldZnode(type, id, name), val.getBytes(ENCODING)); } catch (KeeperException.NodeExistsException e) { - zk.setData(makeFieldZnode(type, id, name), - val.getBytes(ENCODING), - -1); + zk.setData().forPath(makeFieldZnode(type, id, name), val.getBytes(ENCODING)); } } @@ -251,7 +237,7 @@ public void saveField(Type type, String id, String key, String val) @Override public String getField(Type type, String id, String key) { try { - byte[] b = zk.getData(makeFieldZnode(type, id, key), false, null); + byte[] b = zk.getData().forPath(makeFieldZnode(type, id, key)); return new String(b, ENCODING); } catch (Exception e) { return null; @@ -259,26 +245,12 @@ public String getField(Type type, String id, String key) { } @Override - public Map getFields(Type type, String id) { - HashMap map = new HashMap(); - try { - for (String node : zk.getChildren(makeZnode(type, id), false)) { - byte[] b = zk.getData(makeFieldZnode(type, id, node), - false, null); - map.put(node, new String(b, ENCODING)); - } - } catch (Exception e) { - return map; - } - return map; - } - - @Override public boolean delete(Type type, String id) throws NotFoundException { try { - for (String child : zk.getChildren(makeZnode(type, id), false)) { + + for (String child : zk.getChildren().forPath(makeZnode(type, id))) { try { - zk.delete(makeFieldZnode(type, id, child), -1); + zk.delete().forPath(makeFieldZnode(type, id, child)); } catch (Exception e) { // Other nodes may be trying to delete this at the same time, // so just log errors and skip them. @@ -287,7 +259,7 @@ public boolean delete(Type type, String id) throws NotFoundException { } } try { - zk.delete(makeZnode(type, id), -1); + zk.delete().forPath(makeZnode(type, id)); } catch (Exception e) { // Same thing -- might be deleted by other nodes, so just go on. throw new NotFoundException("Couldn't delete " + @@ -302,58 +274,15 @@ public boolean delete(Type type, String id) throws NotFoundException { } @Override - public List getAll() { - ArrayList allNodes = new ArrayList(); - for (Type type : Type.values()) { - allNodes.addAll(getAllForType(type)); - } - return allNodes; - } - - @Override public List getAllForType(Type type) { try { - return zk.getChildren(getPath(type), false); + return zk.getChildren().forPath(getPath(type)); } catch (Exception e) { return new ArrayList(); } } @Override - public List getAllForKey(String key, String value) { - ArrayList allNodes = new ArrayList(); - try { - for (Type type : Type.values()) { - allNodes.addAll(getAllForTypeAndKey(type, key, value)); - } - } catch (Exception e) { - LOG.info("Couldn't find children."); - } - return allNodes; - } - - @Override - public List getAllForTypeAndKey(Type type, String key, String value) { - ArrayList allNodes = new ArrayList(); - try { - for (String id : zk.getChildren(getPath(type), false)) { - for (String field : zk.getChildren(id, false)) { - if (field.endsWith("/" + key)) { - byte[] b = zk.getData(field, false, null); - if (new String(b, ENCODING).equals(value)) { - allNodes.add(id); - } - } - } - } - } catch (Exception e) { - // Log and go to the next type -- this one might not exist - LOG.info("Couldn't find children of " + getPath(type)); - } - return allNodes; - } - - @Override public void openStorage(Configuration config) throws IOException { storage_root = config.get(STORAGE_ROOT); job_path = storage_root + "/jobs"; diff --git a/hwi/pom.xml b/hwi/pom.xml index 35c124b..5db62bb 100644 --- a/hwi/pom.xml +++ b/hwi/pom.xml @@ -19,7 +19,7 @@ org.apache.hive hive - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../pom.xml diff --git a/itests/custom-serde/pom.xml b/itests/custom-serde/pom.xml index cf099c0..6ddc180 100644 --- a/itests/custom-serde/pom.xml +++ b/itests/custom-serde/pom.xml @@ -19,7 +19,7 @@ org.apache.hive hive-it - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../pom.xml diff --git a/itests/custom-serde/src/main/java/org/apache/hadoop/hive/serde2/CustomSerDe1.java b/itests/custom-serde/src/main/java/org/apache/hadoop/hive/serde2/CustomSerDe1.java index e1352c3..c28f096 100644 --- a/itests/custom-serde/src/main/java/org/apache/hadoop/hive/serde2/CustomSerDe1.java +++ b/itests/custom-serde/src/main/java/org/apache/hadoop/hive/serde2/CustomSerDe1.java @@ -34,6 +34,7 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; +@SerDeSpec(schemaProps = {serdeConstants.LIST_COLUMNS, serdeConstants.LIST_COLUMN_TYPES}) public class CustomSerDe1 extends AbstractSerDe { int numColumns; diff --git a/itests/custom-serde/src/main/java/org/apache/hadoop/hive/serde2/CustomSerDe2.java b/itests/custom-serde/src/main/java/org/apache/hadoop/hive/serde2/CustomSerDe2.java index f85439c..05d0590 100644 --- a/itests/custom-serde/src/main/java/org/apache/hadoop/hive/serde2/CustomSerDe2.java +++ b/itests/custom-serde/src/main/java/org/apache/hadoop/hive/serde2/CustomSerDe2.java @@ -35,6 +35,7 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; +@SerDeSpec(schemaProps = {serdeConstants.LIST_COLUMNS, serdeConstants.LIST_COLUMN_TYPES}) public class CustomSerDe2 extends AbstractSerDe { int numColumns; diff --git a/itests/hcatalog-unit/pom.xml b/itests/hcatalog-unit/pom.xml index d1c274c..fabaa94 100644 --- a/itests/hcatalog-unit/pom.xml +++ b/itests/hcatalog-unit/pom.xml @@ -25,7 +25,7 @@ org.apache.hive hive-it - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../pom.xml diff --git a/itests/hive-minikdc/pom.xml b/itests/hive-minikdc/pom.xml index 0bb78a3..f2576bd 100644 --- a/itests/hive-minikdc/pom.xml +++ b/itests/hive-minikdc/pom.xml @@ -19,7 +19,7 @@ org.apache.hive hive-it - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../pom.xml @@ -60,6 +60,13 @@ org.apache.hive + hive-it-unit + ${project.version} + test + tests + + + org.apache.hive hive-jdbc ${project.version} test diff --git a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestHs2HooksWithMiniKdc.java b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestHs2HooksWithMiniKdc.java index e11466d..e089aa9 100644 --- a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestHs2HooksWithMiniKdc.java +++ b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestHs2HooksWithMiniKdc.java @@ -28,73 +28,32 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.ql.hooks.ExecuteWithHookContext; -import org.apache.hadoop.hive.ql.hooks.HookContext; -import org.apache.hadoop.hive.ql.hooks.HookContext.HookType; +import org.apache.hadoop.hive.hooks.TestHs2Hooks.PostExecHook; +import org.apache.hadoop.hive.hooks.TestHs2Hooks.PreExecHook; +import org.apache.hadoop.hive.hooks.TestHs2Hooks.SemanticAnalysisHook; import org.apache.hive.jdbc.miniHS2.MiniHS2; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Tests information retrieved from hooks, in Kerberos mode. */ public class TestHs2HooksWithMiniKdc { - private static final Logger LOG = LoggerFactory.getLogger(TestHs2HooksWithMiniKdc.class); - - public static class PostExecHook implements ExecuteWithHookContext { - private static String userName; - private static String ipAddress; - private static String operation; - private static Throwable error; - - public void run(HookContext hookContext) { - try { - if (hookContext.getHookType().equals(HookType.POST_EXEC_HOOK)) { - ipAddress = hookContext.getIpAddress(); - userName = hookContext.getUserName(); - operation = hookContext.getOperationName(); - } - } catch (Throwable t) { - LOG.error("Error in PostExecHook: " + t, t); - error = t; - } - } - } - - public static class PreExecHook implements ExecuteWithHookContext { - private static String userName; - private static String ipAddress; - private static String operation; - private static Throwable error; - - public void run(HookContext hookContext) { - try { - if (hookContext.getHookType().equals(HookType.PRE_EXEC_HOOK)) { - ipAddress = hookContext.getIpAddress(); - userName = hookContext.getUserName(); - operation = hookContext.getOperationName(); - } - } catch (Throwable t) { - LOG.error("Error in PreExecHook: " + t, t); - error = t; - } - } - } private static MiniHS2 miniHS2 = null; private static MiniHiveKdc miniHiveKdc = null; private static Map confOverlay = new HashMap(); private Connection hs2Conn; @BeforeClass - public static void beforeTest() throws Exception { + public static void setUpBeforeClass() throws Exception { Class.forName(MiniHS2.getJdbcDriverName()); confOverlay.put(ConfVars.POSTEXECHOOKS.varname, PostExecHook.class.getName()); confOverlay.put(ConfVars.PREEXECHOOKS.varname, PreExecHook.class.getName()); + confOverlay.put(ConfVars.SEMANTIC_ANALYZER_HOOK.varname, + SemanticAnalysisHook.class.getName()); HiveConf hiveConf = new HiveConf(); miniHiveKdc = MiniHiveKdc.getMiniHiveKdc(hiveConf); @@ -102,12 +61,30 @@ public static void beforeTest() throws Exception { miniHS2.start(confOverlay); } + @AfterClass + public static void tearDownAfterClass() throws Exception { + miniHS2.stop(); + } + @Before - public void setUp() throws Exception { + public void setUpTest() throws Exception { + PreExecHook.userName = null; + PreExecHook.ipAddress = null; + PreExecHook.operation = null; + PreExecHook.error = null; + PostExecHook.userName = null; + PostExecHook.ipAddress = null; + PostExecHook.operation = null; + PostExecHook.error = null; + SemanticAnalysisHook.userName = null; + SemanticAnalysisHook.ipAddress = null; + SemanticAnalysisHook.command = null; + SemanticAnalysisHook.preAnalyzeError = null; + SemanticAnalysisHook.postAnalyzeError = null; } @After - public void tearDown() throws Exception { + public void tearDownTest() throws Exception { if (hs2Conn != null) { try { hs2Conn.close(); @@ -117,16 +94,11 @@ public void tearDown() throws Exception { } } - @AfterClass - public static void afterTest() throws Exception { - miniHS2.stop(); - } - /** - * Test get IpAddress and username from hook. + * Test that hook context properties are correctly set. */ @Test - public void testIpUserName() throws Throwable { + public void testHookContexts() throws Throwable { miniHiveKdc.loginUser(MiniHiveKdc.HIVE_TEST_USER_1); hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL()); @@ -155,5 +127,24 @@ public void testIpUserName() throws Throwable { Assert.assertEquals(MiniHiveKdc.HIVE_TEST_USER_1, PreExecHook.userName); Assert.assertTrue(PreExecHook.ipAddress, PreExecHook.ipAddress.contains("127.0.0.1")); Assert.assertEquals("SHOWTABLES", PreExecHook.operation); + + error = SemanticAnalysisHook.preAnalyzeError; + if (error != null) { + throw error; + } + error = SemanticAnalysisHook.postAnalyzeError; + if (error != null) { + throw error; + } + + Assert.assertNotNull(SemanticAnalysisHook.ipAddress, + "semantic hook context ipaddress is null"); + Assert.assertNotNull(SemanticAnalysisHook.userName, + "semantic hook context userName is null"); + Assert.assertNotNull(SemanticAnalysisHook.command , + "semantic hook context command is null"); + Assert.assertTrue(SemanticAnalysisHook.ipAddress, + SemanticAnalysisHook.ipAddress.contains("127.0.0.1")); + Assert.assertEquals("show tables", SemanticAnalysisHook.command); } -} +} \ No newline at end of file diff --git a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdc.java b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdc.java index 2355519..6115d4d 100644 --- a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdc.java +++ b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdc.java @@ -19,7 +19,6 @@ package org.apache.hive.minikdc; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -153,7 +152,7 @@ public void testTokenAuth() throws Exception { } /*** - * Negtive test for token based authentication + * Negative test for token based authentication * Verify that a user can't retrieve a token for user that * it's not allowed to impersonate * @throws Exception @@ -163,13 +162,22 @@ public void testNegativeTokenAuth() throws Exception { miniHiveKdc.loginUser(MiniHiveKdc.HIVE_TEST_SUPER_USER); hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL()); - // retrieve token and store in the cache - String token = ((HiveConnection)hs2Conn).getDelegationToken( - MiniHiveKdc.HIVE_TEST_USER_2, MiniHiveKdc.HIVE_SERVICE_PRINCIPAL); - hs2Conn.close(); + try { + // retrieve token and store in the cache + String token = ((HiveConnection)hs2Conn).getDelegationToken( + MiniHiveKdc.HIVE_TEST_USER_2, MiniHiveKdc.HIVE_SERVICE_PRINCIPAL); - assertNull(MiniHiveKdc.HIVE_TEST_SUPER_USER + " shouldn't be allowed to create token for " + - MiniHiveKdc.HIVE_TEST_USER_2, token); + fail(MiniHiveKdc.HIVE_TEST_SUPER_USER + " shouldn't be allowed to retrieve token for " + + MiniHiveKdc.HIVE_TEST_USER_2); + } catch (SQLException e) { + e.printStackTrace(); + // Expected error + assertTrue(e.toString(), e.getMessage().contains("Failed to validate proxy privilege")); + assertTrue(e.getCause().getCause().toString(), + e.getCause().getCause().getMessage().contains("Failed to validate proxy privilege")); + } finally { + hs2Conn.close(); + } } /** @@ -201,7 +209,9 @@ public void testNegativeProxyAuth() throws Exception { + MiniHiveKdc.HIVE_TEST_USER_2); } catch (SQLException e) { // Expected error - assertEquals("08S01", e.getSQLState().trim()); + e.printStackTrace(); + assertTrue(e.getMessage().contains("Failed to validate proxy privilege")); + assertTrue(e.getCause().getCause().getMessage().contains("is not allowed to impersonate")); } } diff --git a/itests/hive-unit-hadoop2/pom.xml b/itests/hive-unit-hadoop2/pom.xml index cbdf506..6c3ded3 100644 --- a/itests/hive-unit-hadoop2/pom.xml +++ b/itests/hive-unit-hadoop2/pom.xml @@ -19,7 +19,7 @@ org.apache.hive hive-it - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../pom.xml @@ -118,6 +118,12 @@ test-jar test + + org.apache.hive + hive-it-unit + ${project.version} + test + org.apache.hadoop diff --git a/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationProviderWithACL.java b/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationProviderWithACL.java index a0d1023..9982195 100644 --- a/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationProviderWithACL.java +++ b/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationProviderWithACL.java @@ -68,6 +68,10 @@ protected HiveConf createHiveConf() throws Exception { conf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, warehouseDir.toString()); conf.setBoolVar(HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS, true); + // Set up scratch directory + Path scratchDir = new Path(new Path(fs.getUri()), "/scratchdir"); + conf.setVar(HiveConf.ConfVars.SCRATCHDIR, scratchDir.toString()); + return conf; } diff --git a/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/thrift/TestHadoop20SAuthBridge.java b/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/thrift/TestHadoop20SAuthBridge.java new file mode 100644 index 0000000..b2bdafa --- /dev/null +++ b/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/thrift/TestHadoop20SAuthBridge.java @@ -0,0 +1,420 @@ +package org.apache.hadoop.hive.thrift; +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.IOException; +import java.net.InetAddress; +import java.net.NetworkInterface; +import java.net.ServerSocket; +import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Enumeration; +import java.util.List; +import java.util.Map; + +import junit.framework.TestCase; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge.Server.ServerMode; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.SaslRpcServer; +import org.apache.hadoop.security.SaslRpcServer.AuthMethod; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; +import org.apache.hadoop.security.authorize.AuthorizationException; +import org.apache.hadoop.security.authorize.DefaultImpersonationProvider; +import org.apache.hadoop.security.authorize.ProxyUsers; +import org.apache.hadoop.security.token.SecretManager.InvalidToken; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager.DelegationTokenInformation; +import org.apache.hadoop.security.token.delegation.DelegationKey; +import org.apache.hadoop.util.StringUtils; +import org.apache.thrift.transport.TSaslServerTransport; +import org.apache.thrift.transport.TTransportException; +import org.apache.thrift.transport.TTransportFactory; + +public class TestHadoop20SAuthBridge extends TestCase { + + /** + * set to true when metastore token manager has intitialized token manager + * through call to HadoopThriftAuthBridge20S.Server.startDelegationTokenSecretManager + */ + static volatile boolean isMetastoreTokenManagerInited; + + private static class MyHadoopThriftAuthBridge20S extends HadoopThriftAuthBridge20S { + @Override + public Server createServer(String keytabFile, String principalConf) + throws TTransportException { + //Create a Server that doesn't interpret any Kerberos stuff + return new Server(); + } + + static class Server extends HadoopThriftAuthBridge20S.Server { + public Server() throws TTransportException { + super(); + } + @Override + public TTransportFactory createTransportFactory(Map saslProps) + throws TTransportException { + TSaslServerTransport.Factory transFactory = + new TSaslServerTransport.Factory(); + transFactory.addServerDefinition(AuthMethod.DIGEST.getMechanismName(), + null, SaslRpcServer.SASL_DEFAULT_REALM, + saslProps, + new SaslDigestCallbackHandler(secretManager)); + + return new TUGIAssumingTransportFactory(transFactory, realUgi); + } + static DelegationTokenStore TOKEN_STORE = new MemoryTokenStore(); + + @Override + protected DelegationTokenStore getTokenStore(Configuration conf) throws IOException { + return TOKEN_STORE; + } + + @Override + public void startDelegationTokenSecretManager(Configuration conf, Object hms, ServerMode sm) + throws IOException{ + super.startDelegationTokenSecretManager(conf, hms, sm); + isMetastoreTokenManagerInited = true; + } + + } + } + + + private HiveConf conf; + + private void configureSuperUserIPAddresses(Configuration conf, + String superUserShortName) throws IOException { + List ipList = new ArrayList(); + Enumeration netInterfaceList = NetworkInterface + .getNetworkInterfaces(); + while (netInterfaceList.hasMoreElements()) { + NetworkInterface inf = netInterfaceList.nextElement(); + Enumeration addrList = inf.getInetAddresses(); + while (addrList.hasMoreElements()) { + InetAddress addr = addrList.nextElement(); + ipList.add(addr.getHostAddress()); + } + } + StringBuilder builder = new StringBuilder(); + for (String ip : ipList) { + builder.append(ip); + builder.append(','); + } + builder.append("127.0.1.1,"); + builder.append(InetAddress.getLocalHost().getCanonicalHostName()); + conf.setStrings(DefaultImpersonationProvider.getProxySuperuserIpConfKey(superUserShortName), + builder.toString()); + } + + public void setup() throws Exception { + isMetastoreTokenManagerInited = false; + int port = findFreePort(); + System.setProperty(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL.varname, + "true"); + System.setProperty(HiveConf.ConfVars.METASTOREURIS.varname, + "thrift://localhost:" + port); + System.setProperty(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, new Path( + System.getProperty("test.build.data", "/tmp")).toString()); + conf = new HiveConf(TestHadoop20SAuthBridge.class); + MetaStoreUtils.startMetaStore(port, new MyHadoopThriftAuthBridge20S()); + } + + /** + * Test delegation token store/load from shared store. + * @throws Exception + */ + public void testDelegationTokenSharedStore() throws Exception { + UserGroupInformation clientUgi = UserGroupInformation.getCurrentUser(); + + TokenStoreDelegationTokenSecretManager tokenManager = + new TokenStoreDelegationTokenSecretManager(0, 60*60*1000, 60*60*1000, 0, + MyHadoopThriftAuthBridge20S.Server.TOKEN_STORE); + // initializes current key + tokenManager.startThreads(); + tokenManager.stopThreads(); + + String tokenStrForm = tokenManager.getDelegationToken(clientUgi.getShortUserName()); + Token t= new Token(); + t.decodeFromUrlString(tokenStrForm); + + //check whether the username in the token is what we expect + DelegationTokenIdentifier d = new DelegationTokenIdentifier(); + d.readFields(new DataInputStream(new ByteArrayInputStream( + t.getIdentifier()))); + assertTrue("Usernames don't match", + clientUgi.getShortUserName().equals(d.getUser().getShortUserName())); + + DelegationTokenInformation tokenInfo = MyHadoopThriftAuthBridge20S.Server.TOKEN_STORE + .getToken(d); + assertNotNull("token not in store", tokenInfo); + assertFalse("duplicate token add", + MyHadoopThriftAuthBridge20S.Server.TOKEN_STORE.addToken(d, tokenInfo)); + + // check keys are copied from token store when token is loaded + TokenStoreDelegationTokenSecretManager anotherManager = + new TokenStoreDelegationTokenSecretManager(0, 0, 0, 0, + MyHadoopThriftAuthBridge20S.Server.TOKEN_STORE); + assertEquals("master keys empty on init", 0, + anotherManager.getAllKeys().length); + assertNotNull("token loaded", + anotherManager.retrievePassword(d)); + anotherManager.renewToken(t, clientUgi.getShortUserName()); + assertEquals("master keys not loaded from store", + MyHadoopThriftAuthBridge20S.Server.TOKEN_STORE.getMasterKeys().length, + anotherManager.getAllKeys().length); + + // cancel the delegation token + tokenManager.cancelDelegationToken(tokenStrForm); + assertNull("token not removed from store after cancel", + MyHadoopThriftAuthBridge20S.Server.TOKEN_STORE.getToken(d)); + assertFalse("token removed (again)", + MyHadoopThriftAuthBridge20S.Server.TOKEN_STORE.removeToken(d)); + try { + anotherManager.retrievePassword(d); + fail("InvalidToken expected after cancel"); + } catch (InvalidToken ex) { + // expected + } + + // token expiration + MyHadoopThriftAuthBridge20S.Server.TOKEN_STORE.addToken(d, + new DelegationTokenInformation(0, t.getPassword())); + assertNotNull(MyHadoopThriftAuthBridge20S.Server.TOKEN_STORE.getToken(d)); + anotherManager.removeExpiredTokens(); + assertNull("Expired token not removed", + MyHadoopThriftAuthBridge20S.Server.TOKEN_STORE.getToken(d)); + + // key expiration - create an already expired key + anotherManager.startThreads(); // generates initial key + anotherManager.stopThreads(); + DelegationKey expiredKey = new DelegationKey(-1, 0, anotherManager.getAllKeys()[0].getKey()); + anotherManager.logUpdateMasterKey(expiredKey); // updates key with sequence number + assertTrue("expired key not in allKeys", + anotherManager.reloadKeys().containsKey(expiredKey.getKeyId())); + anotherManager.rollMasterKeyExt(); + assertFalse("Expired key not removed", + anotherManager.reloadKeys().containsKey(expiredKey.getKeyId())); + } + + public void testSaslWithHiveMetaStore() throws Exception { + setup(); + UserGroupInformation clientUgi = UserGroupInformation.getCurrentUser(); + obtainTokenAndAddIntoUGI(clientUgi, null); + obtainTokenAndAddIntoUGI(clientUgi, "tokenForFooTablePartition"); + } + + public void testMetastoreProxyUser() throws Exception { + setup(); + + final String proxyUserName = "proxyUser"; + //set the configuration up such that proxyUser can act on + //behalf of all users belonging to the group foo_bar_group ( + //a dummy group) + String[] groupNames = + new String[] { "foo_bar_group" }; + setGroupsInConf(groupNames, proxyUserName); + + final UserGroupInformation delegationTokenUser = + UserGroupInformation.getCurrentUser(); + + final UserGroupInformation proxyUserUgi = + UserGroupInformation.createRemoteUser(proxyUserName); + String tokenStrForm = proxyUserUgi.doAs(new PrivilegedExceptionAction() { + public String run() throws Exception { + try { + //Since the user running the test won't belong to a non-existent group + //foo_bar_group, the call to getDelegationTokenStr will fail + return getDelegationTokenStr(delegationTokenUser, proxyUserUgi); + } catch (AuthorizationException ae) { + return null; + } + } + }); + assertTrue("Expected the getDelegationToken call to fail", + tokenStrForm == null); + + //set the configuration up such that proxyUser can act on + //behalf of all users belonging to the real group(s) that the + //user running the test belongs to + setGroupsInConf(UserGroupInformation.getCurrentUser().getGroupNames(), + proxyUserName); + tokenStrForm = proxyUserUgi.doAs(new PrivilegedExceptionAction() { + public String run() throws Exception { + try { + //Since the user running the test belongs to the group + //obtained above the call to getDelegationTokenStr will succeed + return getDelegationTokenStr(delegationTokenUser, proxyUserUgi); + } catch (AuthorizationException ae) { + return null; + } + } + }); + assertTrue("Expected the getDelegationToken call to not fail", + tokenStrForm != null); + Token t= new Token(); + t.decodeFromUrlString(tokenStrForm); + //check whether the username in the token is what we expect + DelegationTokenIdentifier d = new DelegationTokenIdentifier(); + d.readFields(new DataInputStream(new ByteArrayInputStream( + t.getIdentifier()))); + assertTrue("Usernames don't match", + delegationTokenUser.getShortUserName().equals(d.getUser().getShortUserName())); + + } + + private void setGroupsInConf(String[] groupNames, String proxyUserName) + throws IOException { + conf.set( + DefaultImpersonationProvider.getProxySuperuserGroupConfKey(proxyUserName), + StringUtils.join(",", Arrays.asList(groupNames))); + configureSuperUserIPAddresses(conf, proxyUserName); + ProxyUsers.refreshSuperUserGroupsConfiguration(conf); + } + + private String getDelegationTokenStr(UserGroupInformation ownerUgi, + UserGroupInformation realUgi) throws Exception { + //obtain a token by directly invoking the metastore operation(without going + //through the thrift interface). Obtaining a token makes the secret manager + //aware of the user and that it gave the token to the user + //also set the authentication method explicitly to KERBEROS. Since the + //metastore checks whether the authentication method is KERBEROS or not + //for getDelegationToken, and the testcases don't use + //kerberos, this needs to be done + + waitForMetastoreTokenInit(); + + HadoopThriftAuthBridge20S.Server.authenticationMethod + .set(AuthenticationMethod.KERBEROS); + HadoopThriftAuthBridge20S.Server.remoteAddress.set(InetAddress.getLocalHost()); + return + HiveMetaStore.getDelegationToken(ownerUgi.getShortUserName(), + realUgi.getShortUserName()); + } + + /** + * Wait for metastore to have initialized token manager + * This does not have to be done in other metastore test cases as they + * use metastore client which will retry few times on failure + * @throws InterruptedException + */ + private void waitForMetastoreTokenInit() throws InterruptedException { + int waitAttempts = 30; + while(waitAttempts > 0 && !isMetastoreTokenManagerInited){ + Thread.sleep(1000); + waitAttempts--; + } + } + + private void obtainTokenAndAddIntoUGI(UserGroupInformation clientUgi, + String tokenSig) throws Exception { + String tokenStrForm = getDelegationTokenStr(clientUgi, clientUgi); + Token t= new Token(); + t.decodeFromUrlString(tokenStrForm); + + //check whether the username in the token is what we expect + DelegationTokenIdentifier d = new DelegationTokenIdentifier(); + d.readFields(new DataInputStream(new ByteArrayInputStream( + t.getIdentifier()))); + assertTrue("Usernames don't match", + clientUgi.getShortUserName().equals(d.getUser().getShortUserName())); + + if (tokenSig != null) { + conf.set("hive.metastore.token.signature", tokenSig); + t.setService(new Text(tokenSig)); + } + //add the token to the clientUgi for securely talking to the metastore + clientUgi.addToken(t); + //Create the metastore client as the clientUgi. Doing so this + //way will give the client access to the token that was added earlier + //in the clientUgi + HiveMetaStoreClient hiveClient = + clientUgi.doAs(new PrivilegedExceptionAction() { + public HiveMetaStoreClient run() throws Exception { + HiveMetaStoreClient hiveClient = + new HiveMetaStoreClient(conf); + return hiveClient; + } + }); + + assertTrue("Couldn't connect to metastore", hiveClient != null); + + //try out some metastore operations + createDBAndVerifyExistence(hiveClient); + + //check that getDelegationToken fails since we are not authenticating + //over kerberos + boolean pass = false; + try { + hiveClient.getDelegationToken(clientUgi.getUserName()); + } catch (MetaException ex) { + pass = true; + } + assertTrue("Expected the getDelegationToken call to fail", pass == true); + hiveClient.close(); + + //Now cancel the delegation token + HiveMetaStore.cancelDelegationToken(tokenStrForm); + + //now metastore connection should fail + hiveClient = + clientUgi.doAs(new PrivilegedExceptionAction() { + public HiveMetaStoreClient run() { + try { + HiveMetaStoreClient hiveClient = + new HiveMetaStoreClient(conf); + return hiveClient; + } catch (MetaException e) { + return null; + } + } + }); + assertTrue("Expected metastore operations to fail", hiveClient == null); + } + + private void createDBAndVerifyExistence(HiveMetaStoreClient client) + throws Exception { + String dbName = "simpdb"; + Database db = new Database(); + db.setName(dbName); + client.createDatabase(db); + Database db1 = client.getDatabase(dbName); + client.dropDatabase(dbName); + assertTrue("Databases do not match", db1.getName().equals(db.getName())); + } + + private int findFreePort() throws IOException { + ServerSocket socket= new ServerSocket(0); + int port = socket.getLocalPort(); + socket.close(); + return port; + } +} diff --git a/itests/hive-unit-hadoop2/src/test/java/org/apache/hive/jdbc/TestSchedulerQueue.java b/itests/hive-unit-hadoop2/src/test/java/org/apache/hive/jdbc/TestSchedulerQueue.java new file mode 100644 index 0000000..79878ba --- /dev/null +++ b/itests/hive-unit-hadoop2/src/test/java/org/apache/hive/jdbc/TestSchedulerQueue.java @@ -0,0 +1,120 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.jdbc; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.HashMap; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hive.jdbc.miniHS2.MiniHS2; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestSchedulerQueue { + + private MiniHS2 miniHS2 = null; + private static HiveConf conf = new HiveConf(); + private Connection hs2Conn = null; + + @BeforeClass + public static void beforeTest() throws Exception { + Class.forName(MiniHS2.getJdbcDriverName()); + } + + @Before + public void setUp() throws Exception { + DriverManager.setLoginTimeout(0); + miniHS2 = new MiniHS2(conf, true); + miniHS2.setConfProperty(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname, "false"); + miniHS2.setConfProperty(HiveConf.ConfVars.HIVE_SERVER2_MAP_FAIR_SCHEDULER_QUEUE.varname, + "true"); + miniHS2.setConfProperty(YarnConfiguration.RM_SCHEDULER, + "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler"); + miniHS2.start(new HashMap()); + } + + @After + public void tearDown() throws Exception { + if (hs2Conn != null) { + hs2Conn.close(); + } + if (miniHS2 != null && miniHS2.isStarted()) { + miniHS2.stop(); + } + System.clearProperty("mapreduce.job.queuename"); + } + + /** + * Verify: + * Test is running with MR2 and queue mapping defaults are set. + * Queue mapping is set for the connected user. + * + * @throws Exception + */ + @Test + public void testFairSchedulerQueueMapping() throws Exception { + hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL(), "user1", "bar"); + verifyProperty("mapreduce.framework.name", "yarn"); + verifyProperty(HiveConf.ConfVars.HIVE_SERVER2_MAP_FAIR_SCHEDULER_QUEUE.varname, + "true"); + verifyProperty(YarnConfiguration.RM_SCHEDULER, + "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler"); + verifyProperty("mapreduce.job.queuename", "root.user1"); + } + + /** + * Verify that the queue refresh doesn't happen when configured to be off. + * + * @throws Exception + */ + @Test + public void testQueueMappingCheckDisabled() throws Exception { + miniHS2.setConfProperty( + HiveConf.ConfVars.HIVE_SERVER2_MAP_FAIR_SCHEDULER_QUEUE.varname, "false"); + hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL(), "user1", "bar"); + verifyProperty(HiveConf.ConfVars.HIVE_SERVER2_MAP_FAIR_SCHEDULER_QUEUE.varname, + "false"); + verifyProperty("mapreduce.job.queuename", YarnConfiguration.DEFAULT_QUEUE_NAME); + } + + /** + * Verify that the given property contains the expected value. + * + * @param propertyName + * @param expectedValue + * @throws Exception + */ + private void verifyProperty(String propertyName, String expectedValue) throws Exception { + Statement stmt = hs2Conn .createStatement(); + ResultSet res = stmt.executeQuery("set " + propertyName); + assertTrue(res.next()); + String results[] = res.getString(1).split("="); + assertEquals("Property should be set", results.length, 2); + assertEquals("Property should be set", expectedValue, results[1]); + } +} diff --git a/itests/hive-unit/pom.xml b/itests/hive-unit/pom.xml index 077631a..f9f59c9 100644 --- a/itests/hive-unit/pom.xml +++ b/itests/hive-unit/pom.xml @@ -19,7 +19,7 @@ org.apache.hive hive-it - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../pom.xml diff --git a/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java b/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java index ffad413..32b971c 100644 --- a/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java +++ b/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java @@ -176,7 +176,7 @@ private MiniHS2(HiveConf hiveConf, boolean useMiniMR, boolean useMiniKdc, baseDfsDir = new Path(new Path(fs.getUri()), "/base"); } else { fs = FileSystem.getLocal(hiveConf); - baseDfsDir = new Path("file://"+ baseDir.getPath()); + baseDfsDir = new Path("file://"+ baseDir.toURI().getPath()); } if (useMiniKdc) { hiveConf.setVar(ConfVars.HIVE_SERVER2_KERBEROS_PRINCIPAL, serverPrincipal); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java index 0676758..5696076 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java @@ -19,8 +19,11 @@ //The tests here are heavily based on some timing, so there is some chance to fail. package org.apache.hadoop.hive.hooks; -import java.util.Properties; +import java.io.Serializable; +import java.lang.Override; import java.sql.Statement; +import java.util.List; +import java.util.Properties; import junit.framework.Assert; @@ -28,9 +31,15 @@ import org.apache.hadoop.hive.ql.hooks.ExecuteWithHookContext; import org.apache.hadoop.hive.ql.hooks.HookContext; import org.apache.hadoop.hive.ql.hooks.HookContext.HookType; +import org.apache.hadoop.hive.ql.parse.ASTNode; +import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHook; +import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHookContext; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hive.jdbc.HiveConnection; import org.apache.hive.service.server.HiveServer2; import org.junit.AfterClass; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.slf4j.Logger; @@ -44,10 +53,10 @@ private static HiveServer2 hiveServer2; public static class PostExecHook implements ExecuteWithHookContext { - private static String userName; - private static String ipAddress; - private static String operation; - private static Throwable error; + public static String userName; + public static String ipAddress; + public static String operation; + public static Throwable error; public void run(HookContext hookContext) { try { @@ -64,10 +73,10 @@ public void run(HookContext hookContext) { } public static class PreExecHook implements ExecuteWithHookContext { - private static String userName; - private static String ipAddress; - private static String operation; - private static Throwable error; + public static String userName; + public static String ipAddress; + public static String operation; + public static Throwable error; public void run(HookContext hookContext) { try { @@ -83,6 +92,41 @@ public void run(HookContext hookContext) { } } + public static class SemanticAnalysisHook implements HiveSemanticAnalyzerHook { + public static String userName; + public static String command; + public static String ipAddress; + public static Throwable preAnalyzeError; + public static Throwable postAnalyzeError; + + @Override + public ASTNode preAnalyze(HiveSemanticAnalyzerHookContext context, + ASTNode ast) throws SemanticException { + try { + userName = context.getUserName(); + ipAddress = context.getIpAddress(); + command = context.getCommand(); + } catch (Throwable t) { + LOG.error("Error in semantic analysis hook preAnalyze: " + t, t); + preAnalyzeError = t; + } + return ast; + } + + @Override + public void postAnalyze(HiveSemanticAnalyzerHookContext context, + List> rootTasks) throws SemanticException { + try { + userName = context.getUserName(); + ipAddress = context.getIpAddress(); + command = context.getCommand(); + } catch (Throwable t) { + LOG.error("Error in semantic analysis hook postAnalyze: " + t, t); + postAnalyzeError = t; + } + } + } + /** * @throws java.lang.Exception */ @@ -93,6 +137,8 @@ public static void setUpBeforeClass() throws Exception { PreExecHook.class.getName()); hiveConf.setVar(HiveConf.ConfVars.POSTEXECHOOKS, PostExecHook.class.getName()); + hiveConf.setVar(HiveConf.ConfVars.SEMANTIC_ANALYZER_HOOK, + SemanticAnalysisHook.class.getName()); hiveServer2 = new HiveServer2(); hiveServer2.init(hiveConf); @@ -107,16 +153,32 @@ public static void tearDownAfterClass() throws Exception { } } + @Before + public void setUpTest() throws Exception { + PreExecHook.userName = null; + PreExecHook.ipAddress = null; + PreExecHook.operation = null; + PreExecHook.error = null; + PostExecHook.userName = null; + PostExecHook.ipAddress = null; + PostExecHook.operation = null; + PostExecHook.error = null; + SemanticAnalysisHook.userName = null; + SemanticAnalysisHook.ipAddress = null; + SemanticAnalysisHook.command = null; + SemanticAnalysisHook.preAnalyzeError = null; + SemanticAnalysisHook.postAnalyzeError = null; + } + /** - * Test get IpAddress and username from hook. + * Test that hook context properties are correctly set. */ @Test - public void testIpUserName() throws Throwable { + public void testHookContexts() throws Throwable { Properties connProp = new Properties(); connProp.setProperty("user", System.getProperty("user.name")); connProp.setProperty("password", ""); HiveConnection connection = new HiveConnection("jdbc:hive2://localhost:10000/default", connProp); - Statement stmt = connection.createStatement(); stmt.executeQuery("show databases"); stmt.executeQuery("show tables"); @@ -142,6 +204,24 @@ public void testIpUserName() throws Throwable { Assert.assertNotNull(PreExecHook.operation , "operation is null"); Assert.assertTrue(PreExecHook.ipAddress, PreExecHook.ipAddress.contains("127.0.0.1")); Assert.assertEquals("SHOWTABLES", PreExecHook.operation); + + error = SemanticAnalysisHook.preAnalyzeError; + if (error != null) { + throw error; + } + error = SemanticAnalysisHook.postAnalyzeError; + if (error != null) { + throw error; + } + + Assert.assertNotNull(SemanticAnalysisHook.ipAddress, + "semantic hook context ipaddress is null"); + Assert.assertNotNull(SemanticAnalysisHook.userName, + "semantic hook context userName is null"); + Assert.assertNotNull(SemanticAnalysisHook.command , + "semantic hook context command is null"); + Assert.assertTrue(SemanticAnalysisHook.ipAddress, + SemanticAnalysisHook.ipAddress.contains("127.0.0.1")); + Assert.assertEquals("show tables", SemanticAnalysisHook.command); } } - diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java index 3b5f65f..4dbd26a 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java @@ -29,26 +29,33 @@ import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.Index; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.PartitionEventType; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.events.AddIndexEvent; import org.apache.hadoop.hive.metastore.events.AddPartitionEvent; +import org.apache.hadoop.hive.metastore.events.AlterIndexEvent; import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent; import org.apache.hadoop.hive.metastore.events.AlterTableEvent; import org.apache.hadoop.hive.metastore.events.ConfigChangeEvent; import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent; import org.apache.hadoop.hive.metastore.events.CreateTableEvent; import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent; +import org.apache.hadoop.hive.metastore.events.DropIndexEvent; import org.apache.hadoop.hive.metastore.events.DropPartitionEvent; import org.apache.hadoop.hive.metastore.events.DropTableEvent; import org.apache.hadoop.hive.metastore.events.ListenerEvent; import org.apache.hadoop.hive.metastore.events.LoadPartitionDoneEvent; +import org.apache.hadoop.hive.metastore.events.PreAddIndexEvent; import org.apache.hadoop.hive.metastore.events.PreAddPartitionEvent; +import org.apache.hadoop.hive.metastore.events.PreAlterIndexEvent; import org.apache.hadoop.hive.metastore.events.PreAlterPartitionEvent; import org.apache.hadoop.hive.metastore.events.PreAlterTableEvent; import org.apache.hadoop.hive.metastore.events.PreCreateDatabaseEvent; import org.apache.hadoop.hive.metastore.events.PreCreateTableEvent; import org.apache.hadoop.hive.metastore.events.PreDropDatabaseEvent; +import org.apache.hadoop.hive.metastore.events.PreDropIndexEvent; import org.apache.hadoop.hive.metastore.events.PreDropPartitionEvent; import org.apache.hadoop.hive.metastore.events.PreDropTableEvent; import org.apache.hadoop.hive.metastore.events.PreEventContext; @@ -183,12 +190,35 @@ private void validateDropDb(Database expectedDb, Database actualDb) { assertEquals(expectedDb, actualDb); } + private void validateIndex(Index expectedIndex, Index actualIndex) { + assertEquals(expectedIndex.getDbName(), actualIndex.getDbName()); + assertEquals(expectedIndex.getIndexName(), actualIndex.getIndexName()); + assertEquals(expectedIndex.getIndexHandlerClass(), actualIndex.getIndexHandlerClass()); + assertEquals(expectedIndex.getOrigTableName(), actualIndex.getOrigTableName()); + assertEquals(expectedIndex.getIndexTableName(), actualIndex.getIndexTableName()); + assertEquals(expectedIndex.getSd().getLocation(), actualIndex.getSd().getLocation()); + } + + private void validateAddIndex(Index expectedIndex, Index actualIndex) { + validateIndex(expectedIndex, actualIndex); + } + + private void validateAlterIndex(Index expectedOldIndex, Index actualOldIndex, + Index expectedNewIndex, Index actualNewIndex) { + validateIndex(expectedOldIndex, actualOldIndex); + validateIndex(expectedNewIndex, actualNewIndex); + } + + private void validateDropIndex(Index expectedIndex, Index actualIndex) { + validateIndex(expectedIndex, actualIndex); + } + public void testListener() throws Exception { int listSize = 0; List notifyList = DummyListener.notifyList; - assertEquals(notifyList.size(), listSize); List preNotifyList = DummyPreListener.notifyList; + assertEquals(notifyList.size(), listSize); assertEquals(preNotifyList.size(), listSize); driver.run("create database " + dbName); @@ -216,6 +246,48 @@ public void testListener() throws Exception { assert tblEvent.getStatus(); validateCreateTable(tbl, tblEvent.getTable()); + driver.run("create index tmptbl_i on table tmptbl(a) as 'compact' " + + "WITH DEFERRED REBUILD IDXPROPERTIES ('prop1'='val1', 'prop2'='val2')"); + listSize += 2; // creates index table internally + assertEquals(notifyList.size(), listSize); + + AddIndexEvent addIndexEvent = (AddIndexEvent)notifyList.get(listSize - 1); + assert addIndexEvent.getStatus(); + PreAddIndexEvent preAddIndexEvent = (PreAddIndexEvent)(preNotifyList.get(preNotifyList.size() - 3)); + + Index oldIndex = msc.getIndex(dbName, "tmptbl", "tmptbl_i"); + + validateAddIndex(oldIndex, addIndexEvent.getIndex()); + + validateAddIndex(oldIndex, preAddIndexEvent.getIndex()); + + driver.run("alter index tmptbl_i on tmptbl set IDXPROPERTIES " + + "('prop1'='val1_new', 'prop3'='val3')"); + listSize++; + assertEquals(notifyList.size(), listSize); + + Index newIndex = msc.getIndex(dbName, "tmptbl", "tmptbl_i"); + + AlterIndexEvent alterIndexEvent = (AlterIndexEvent) notifyList.get(listSize - 1); + assert alterIndexEvent.getStatus(); + validateAlterIndex(oldIndex, alterIndexEvent.getOldIndex(), + newIndex, alterIndexEvent.getNewIndex()); + + PreAlterIndexEvent preAlterIndexEvent = (PreAlterIndexEvent) (preNotifyList.get(preNotifyList.size() - 1)); + validateAlterIndex(oldIndex, preAlterIndexEvent.getOldIndex(), + newIndex, preAlterIndexEvent.getNewIndex()); + + driver.run("drop index tmptbl_i on tmptbl"); + listSize++; + assertEquals(notifyList.size(), listSize); + + DropIndexEvent dropIndexEvent = (DropIndexEvent) notifyList.get(listSize - 1); + assert dropIndexEvent.getStatus(); + validateDropIndex(newIndex, dropIndexEvent.getIndex()); + + PreDropIndexEvent preDropIndexEvent = (PreDropIndexEvent) (preNotifyList.get(preNotifyList.size() - 1)); + validateDropIndex(newIndex, preDropIndexEvent.getIndex()); + driver.run("alter table tmptbl add partition (b='2011')"); listSize++; assertEquals(notifyList.size(), listSize); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java index 7f8ca87..8c87059 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/FolderPermissionBase.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.shims.HadoopShims.MiniDFSShim; import org.apache.hadoop.hive.shims.ShimLoader; +import org.junit.Before; import org.junit.Test; /** @@ -83,10 +84,15 @@ public static void baseSetup() throws Exception { fs.mkdirs(warehouseDir); conf.setVar(ConfVars.METASTOREWAREHOUSE, warehouseDir.toString()); + // Assuming the tests are run either in C or D drive in Windows OS! dataFileDir = conf.get("test.data.files").replace('\\', '/') - .replace("c:", ""); + .replace("c:", "").replace("C:", "").replace("D:", "").replace("d:", ""); dataFilePath = new Path(dataFileDir, "kv1.txt"); + // Set up scratch directory + Path scratchDir = new Path(baseDfsDir, "scratchdir"); + conf.setVar(HiveConf.ConfVars.SCRATCHDIR, scratchDir.toString()); + //set hive conf vars conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); conf.setBoolVar(HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS, true); @@ -114,6 +120,11 @@ private static void setupDataTable() throws Exception { Assert.assertEquals(0,ret.getResponseCode()); } + @Before + public void setupBeforeTest() throws Exception { + driver.run("USE default"); + } + @Test public void testCreateDb() throws Exception { //see if db inherits permission from warehouse directory. diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java index 0c56132..95f1c39 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java @@ -230,8 +230,9 @@ public void testStatsAfterCompactionPartTbl() throws Exception { t.setThreadId((int) t.getId()); t.setHiveConf(conf); MetaStoreThread.BooleanPointer stop = new MetaStoreThread.BooleanPointer(); + MetaStoreThread.BooleanPointer looped = new MetaStoreThread.BooleanPointer(); stop.boolVal = true; - t.init(stop); + t.init(stop, looped); t.run(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List compacts = rsp.getCompacts(); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestDBTokenStore.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestDBTokenStore.java index a60ad70..0b61a62 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestDBTokenStore.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestDBTokenStore.java @@ -37,7 +37,7 @@ public void testDBTokenStore() throws TokenStoreException, MetaException, IOException { DelegationTokenStore ts = new DBTokenStore(); - ts.setStore(new HMSHandler("Test handler")); + ts.init(new HMSHandler("Test handler").getMS(), null); assertEquals(0, ts.getMasterKeys().length); assertEquals(false,ts.removeMasterKey(-1)); try{ diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestHadoop20SAuthBridge.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestHadoop20SAuthBridge.java index f40b7da..e69de29 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestHadoop20SAuthBridge.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestHadoop20SAuthBridge.java @@ -1,419 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.thrift; - -import java.io.ByteArrayInputStream; -import java.io.DataInputStream; -import java.io.IOException; -import java.net.InetAddress; -import java.net.NetworkInterface; -import java.net.ServerSocket; -import java.security.PrivilegedExceptionAction; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Enumeration; -import java.util.List; -import java.util.Map; - -import junit.framework.TestCase; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.HiveMetaStore; -import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; -import org.apache.hadoop.hive.metastore.MetaStoreUtils; -import org.apache.hadoop.hive.metastore.api.Database; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.security.SaslRpcServer; -import org.apache.hadoop.security.SaslRpcServer.AuthMethod; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; -import org.apache.hadoop.security.authorize.AuthorizationException; -import org.apache.hadoop.security.authorize.DefaultImpersonationProvider; -import org.apache.hadoop.security.authorize.ProxyUsers; -import org.apache.hadoop.security.token.SecretManager.InvalidToken; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager.DelegationTokenInformation; -import org.apache.hadoop.security.token.delegation.DelegationKey; -import org.apache.hadoop.util.StringUtils; -import org.apache.thrift.transport.TSaslServerTransport; -import org.apache.thrift.transport.TTransportException; -import org.apache.thrift.transport.TTransportFactory; - -public class TestHadoop20SAuthBridge extends TestCase { - - /** - * set to true when metastore token manager has intitialized token manager - * through call to HadoopThriftAuthBridge20S.Server.startDelegationTokenSecretManager - */ - static volatile boolean isMetastoreTokenManagerInited; - - private static class MyHadoopThriftAuthBridge20S extends HadoopThriftAuthBridge20S { - @Override - public Server createServer(String keytabFile, String principalConf) - throws TTransportException { - //Create a Server that doesn't interpret any Kerberos stuff - return new Server(); - } - - static class Server extends HadoopThriftAuthBridge20S.Server { - public Server() throws TTransportException { - super(); - } - @Override - public TTransportFactory createTransportFactory(Map saslProps) - throws TTransportException { - TSaslServerTransport.Factory transFactory = - new TSaslServerTransport.Factory(); - transFactory.addServerDefinition(AuthMethod.DIGEST.getMechanismName(), - null, SaslRpcServer.SASL_DEFAULT_REALM, - saslProps, - new SaslDigestCallbackHandler(secretManager)); - - return new TUGIAssumingTransportFactory(transFactory, realUgi); - } - static DelegationTokenStore TOKEN_STORE = new MemoryTokenStore(); - - @Override - protected DelegationTokenStore getTokenStore(Configuration conf) throws IOException { - return TOKEN_STORE; - } - - @Override - public void startDelegationTokenSecretManager(Configuration conf, Object hms) - throws IOException{ - super.startDelegationTokenSecretManager(conf, hms); - isMetastoreTokenManagerInited = true; - } - - } - } - - - private HiveConf conf; - - private void configureSuperUserIPAddresses(Configuration conf, - String superUserShortName) throws IOException { - List ipList = new ArrayList(); - Enumeration netInterfaceList = NetworkInterface - .getNetworkInterfaces(); - while (netInterfaceList.hasMoreElements()) { - NetworkInterface inf = netInterfaceList.nextElement(); - Enumeration addrList = inf.getInetAddresses(); - while (addrList.hasMoreElements()) { - InetAddress addr = addrList.nextElement(); - ipList.add(addr.getHostAddress()); - } - } - StringBuilder builder = new StringBuilder(); - for (String ip : ipList) { - builder.append(ip); - builder.append(','); - } - builder.append("127.0.1.1,"); - builder.append(InetAddress.getLocalHost().getCanonicalHostName()); - conf.setStrings(DefaultImpersonationProvider.getProxySuperuserIpConfKey(superUserShortName), - builder.toString()); - } - - public void setup() throws Exception { - isMetastoreTokenManagerInited = false; - int port = findFreePort(); - System.setProperty(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL.varname, - "true"); - System.setProperty(HiveConf.ConfVars.METASTOREURIS.varname, - "thrift://localhost:" + port); - System.setProperty(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, new Path( - System.getProperty("test.build.data", "/tmp")).toString()); - conf = new HiveConf(TestHadoop20SAuthBridge.class); - MetaStoreUtils.startMetaStore(port, new MyHadoopThriftAuthBridge20S()); - } - - /** - * Test delegation token store/load from shared store. - * @throws Exception - */ - public void testDelegationTokenSharedStore() throws Exception { - UserGroupInformation clientUgi = UserGroupInformation.getCurrentUser(); - - TokenStoreDelegationTokenSecretManager tokenManager = - new TokenStoreDelegationTokenSecretManager(0, 60*60*1000, 60*60*1000, 0, - MyHadoopThriftAuthBridge20S.Server.TOKEN_STORE); - // initializes current key - tokenManager.startThreads(); - tokenManager.stopThreads(); - - String tokenStrForm = tokenManager.getDelegationToken(clientUgi.getShortUserName()); - Token t= new Token(); - t.decodeFromUrlString(tokenStrForm); - - //check whether the username in the token is what we expect - DelegationTokenIdentifier d = new DelegationTokenIdentifier(); - d.readFields(new DataInputStream(new ByteArrayInputStream( - t.getIdentifier()))); - assertTrue("Usernames don't match", - clientUgi.getShortUserName().equals(d.getUser().getShortUserName())); - - DelegationTokenInformation tokenInfo = MyHadoopThriftAuthBridge20S.Server.TOKEN_STORE - .getToken(d); - assertNotNull("token not in store", tokenInfo); - assertFalse("duplicate token add", - MyHadoopThriftAuthBridge20S.Server.TOKEN_STORE.addToken(d, tokenInfo)); - - // check keys are copied from token store when token is loaded - TokenStoreDelegationTokenSecretManager anotherManager = - new TokenStoreDelegationTokenSecretManager(0, 0, 0, 0, - MyHadoopThriftAuthBridge20S.Server.TOKEN_STORE); - assertEquals("master keys empty on init", 0, - anotherManager.getAllKeys().length); - assertNotNull("token loaded", - anotherManager.retrievePassword(d)); - anotherManager.renewToken(t, clientUgi.getShortUserName()); - assertEquals("master keys not loaded from store", - MyHadoopThriftAuthBridge20S.Server.TOKEN_STORE.getMasterKeys().length, - anotherManager.getAllKeys().length); - - // cancel the delegation token - tokenManager.cancelDelegationToken(tokenStrForm); - assertNull("token not removed from store after cancel", - MyHadoopThriftAuthBridge20S.Server.TOKEN_STORE.getToken(d)); - assertFalse("token removed (again)", - MyHadoopThriftAuthBridge20S.Server.TOKEN_STORE.removeToken(d)); - try { - anotherManager.retrievePassword(d); - fail("InvalidToken expected after cancel"); - } catch (InvalidToken ex) { - // expected - } - - // token expiration - MyHadoopThriftAuthBridge20S.Server.TOKEN_STORE.addToken(d, - new DelegationTokenInformation(0, t.getPassword())); - assertNotNull(MyHadoopThriftAuthBridge20S.Server.TOKEN_STORE.getToken(d)); - anotherManager.removeExpiredTokens(); - assertNull("Expired token not removed", - MyHadoopThriftAuthBridge20S.Server.TOKEN_STORE.getToken(d)); - - // key expiration - create an already expired key - anotherManager.startThreads(); // generates initial key - anotherManager.stopThreads(); - DelegationKey expiredKey = new DelegationKey(-1, 0, anotherManager.getAllKeys()[0].getKey()); - anotherManager.logUpdateMasterKey(expiredKey); // updates key with sequence number - assertTrue("expired key not in allKeys", - anotherManager.reloadKeys().containsKey(expiredKey.getKeyId())); - anotherManager.rollMasterKeyExt(); - assertFalse("Expired key not removed", - anotherManager.reloadKeys().containsKey(expiredKey.getKeyId())); - } - - public void testSaslWithHiveMetaStore() throws Exception { - setup(); - UserGroupInformation clientUgi = UserGroupInformation.getCurrentUser(); - obtainTokenAndAddIntoUGI(clientUgi, null); - obtainTokenAndAddIntoUGI(clientUgi, "tokenForFooTablePartition"); - } - - public void testMetastoreProxyUser() throws Exception { - setup(); - - final String proxyUserName = "proxyUser"; - //set the configuration up such that proxyUser can act on - //behalf of all users belonging to the group foo_bar_group ( - //a dummy group) - String[] groupNames = - new String[] { "foo_bar_group" }; - setGroupsInConf(groupNames, proxyUserName); - - final UserGroupInformation delegationTokenUser = - UserGroupInformation.getCurrentUser(); - - final UserGroupInformation proxyUserUgi = - UserGroupInformation.createRemoteUser(proxyUserName); - String tokenStrForm = proxyUserUgi.doAs(new PrivilegedExceptionAction() { - public String run() throws Exception { - try { - //Since the user running the test won't belong to a non-existent group - //foo_bar_group, the call to getDelegationTokenStr will fail - return getDelegationTokenStr(delegationTokenUser, proxyUserUgi); - } catch (AuthorizationException ae) { - return null; - } - } - }); - assertTrue("Expected the getDelegationToken call to fail", - tokenStrForm == null); - - //set the configuration up such that proxyUser can act on - //behalf of all users belonging to the real group(s) that the - //user running the test belongs to - setGroupsInConf(UserGroupInformation.getCurrentUser().getGroupNames(), - proxyUserName); - tokenStrForm = proxyUserUgi.doAs(new PrivilegedExceptionAction() { - public String run() throws Exception { - try { - //Since the user running the test belongs to the group - //obtained above the call to getDelegationTokenStr will succeed - return getDelegationTokenStr(delegationTokenUser, proxyUserUgi); - } catch (AuthorizationException ae) { - return null; - } - } - }); - assertTrue("Expected the getDelegationToken call to not fail", - tokenStrForm != null); - Token t= new Token(); - t.decodeFromUrlString(tokenStrForm); - //check whether the username in the token is what we expect - DelegationTokenIdentifier d = new DelegationTokenIdentifier(); - d.readFields(new DataInputStream(new ByteArrayInputStream( - t.getIdentifier()))); - assertTrue("Usernames don't match", - delegationTokenUser.getShortUserName().equals(d.getUser().getShortUserName())); - - } - - private void setGroupsInConf(String[] groupNames, String proxyUserName) - throws IOException { - conf.set( - DefaultImpersonationProvider.getProxySuperuserGroupConfKey(proxyUserName), - StringUtils.join(",", Arrays.asList(groupNames))); - configureSuperUserIPAddresses(conf, proxyUserName); - ProxyUsers.refreshSuperUserGroupsConfiguration(conf); - } - - private String getDelegationTokenStr(UserGroupInformation ownerUgi, - UserGroupInformation realUgi) throws Exception { - //obtain a token by directly invoking the metastore operation(without going - //through the thrift interface). Obtaining a token makes the secret manager - //aware of the user and that it gave the token to the user - //also set the authentication method explicitly to KERBEROS. Since the - //metastore checks whether the authentication method is KERBEROS or not - //for getDelegationToken, and the testcases don't use - //kerberos, this needs to be done - - waitForMetastoreTokenInit(); - - HadoopThriftAuthBridge20S.Server.authenticationMethod - .set(AuthenticationMethod.KERBEROS); - HadoopThriftAuthBridge20S.Server.remoteAddress.set(InetAddress.getLocalHost()); - return - HiveMetaStore.getDelegationToken(ownerUgi.getShortUserName(), - realUgi.getShortUserName()); - } - - /** - * Wait for metastore to have initialized token manager - * This does not have to be done in other metastore test cases as they - * use metastore client which will retry few times on failure - * @throws InterruptedException - */ - private void waitForMetastoreTokenInit() throws InterruptedException { - int waitAttempts = 30; - while(waitAttempts > 0 && !isMetastoreTokenManagerInited){ - Thread.sleep(1000); - waitAttempts--; - } - } - - private void obtainTokenAndAddIntoUGI(UserGroupInformation clientUgi, - String tokenSig) throws Exception { - String tokenStrForm = getDelegationTokenStr(clientUgi, clientUgi); - Token t= new Token(); - t.decodeFromUrlString(tokenStrForm); - - //check whether the username in the token is what we expect - DelegationTokenIdentifier d = new DelegationTokenIdentifier(); - d.readFields(new DataInputStream(new ByteArrayInputStream( - t.getIdentifier()))); - assertTrue("Usernames don't match", - clientUgi.getShortUserName().equals(d.getUser().getShortUserName())); - - if (tokenSig != null) { - conf.set("hive.metastore.token.signature", tokenSig); - t.setService(new Text(tokenSig)); - } - //add the token to the clientUgi for securely talking to the metastore - clientUgi.addToken(t); - //Create the metastore client as the clientUgi. Doing so this - //way will give the client access to the token that was added earlier - //in the clientUgi - HiveMetaStoreClient hiveClient = - clientUgi.doAs(new PrivilegedExceptionAction() { - public HiveMetaStoreClient run() throws Exception { - HiveMetaStoreClient hiveClient = - new HiveMetaStoreClient(conf); - return hiveClient; - } - }); - - assertTrue("Couldn't connect to metastore", hiveClient != null); - - //try out some metastore operations - createDBAndVerifyExistence(hiveClient); - - //check that getDelegationToken fails since we are not authenticating - //over kerberos - boolean pass = false; - try { - hiveClient.getDelegationToken(clientUgi.getUserName()); - } catch (MetaException ex) { - pass = true; - } - assertTrue("Expected the getDelegationToken call to fail", pass == true); - hiveClient.close(); - - //Now cancel the delegation token - HiveMetaStore.cancelDelegationToken(tokenStrForm); - - //now metastore connection should fail - hiveClient = - clientUgi.doAs(new PrivilegedExceptionAction() { - public HiveMetaStoreClient run() { - try { - HiveMetaStoreClient hiveClient = - new HiveMetaStoreClient(conf); - return hiveClient; - } catch (MetaException e) { - return null; - } - } - }); - assertTrue("Expected metastore operations to fail", hiveClient == null); - } - - private void createDBAndVerifyExistence(HiveMetaStoreClient client) - throws Exception { - String dbName = "simpdb"; - Database db = new Database(); - db.setName(dbName); - client.createDatabase(db); - Database db1 = client.getDatabase(dbName); - client.dropDatabase(dbName); - assertTrue("Databases do not match", db1.getName().equals(db.getName())); - } - - private int findFreePort() throws IOException { - ServerSocket socket= new ServerSocket(0); - int port = socket.getLocalPort(); - socket.close(); - return port; - } -} diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestZooKeeperTokenStore.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestZooKeeperTokenStore.java index 83a80b4..26d4d97 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestZooKeeperTokenStore.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestZooKeeperTokenStore.java @@ -24,25 +24,28 @@ import junit.framework.TestCase; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; +import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge.Server.ServerMode; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager.DelegationTokenInformation; import org.apache.hadoop.security.token.delegation.HiveDelegationTokenSupport; import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.ZooKeeper; import org.apache.zookeeper.data.ACL; -import org.apache.zookeeper.data.Stat; import org.junit.Assert; public class TestZooKeeperTokenStore extends TestCase { private MiniZooKeeperCluster zkCluster = null; - private ZooKeeper zkClient = null; + private CuratorFramework zkClient = null; private int zkPort = -1; private ZooKeeperTokenStore ts; // connect timeout large enough for slower test environments private final int connectTimeoutMillis = 30000; + private final int sessionTimeoutMillis = 3000; @Override protected void setUp() throws Exception { @@ -53,8 +56,10 @@ protected void setUp() throws Exception { this.zkCluster = new MiniZooKeeperCluster(); this.zkPort = this.zkCluster.startup(zkDataDir); - this.zkClient = ZooKeeperTokenStore.createConnectedClient("localhost:" + zkPort, 3000, - connectTimeoutMillis); + this.zkClient = CuratorFrameworkFactory.builder().connectString("localhost:" + zkPort) + .sessionTimeoutMs(sessionTimeoutMillis).connectionTimeoutMs(connectTimeoutMillis) + .retryPolicy(new ExponentialBackoffRetry(1000, 3)).build(); + this.zkClient.start(); } @Override @@ -84,14 +89,16 @@ private Configuration createConf(String zkPath) { public void testTokenStorage() throws Exception { String ZK_PATH = "/zktokenstore-testTokenStorage"; ts = new ZooKeeperTokenStore(); - ts.setConf(createConf(ZK_PATH)); + Configuration conf = createConf(ZK_PATH); + conf.set(HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_ACL, "world:anyone:cdrwa"); + ts.setConf(conf); + ts.init(null, ServerMode.METASTORE); + + String metastore_zk_path = ZK_PATH + ServerMode.METASTORE; int keySeq = ts.addMasterKey("key1Data"); - byte[] keyBytes = zkClient.getData( - ZK_PATH - + "/keys/" - + String.format(ZooKeeperTokenStore.ZK_SEQ_FORMAT, - keySeq), false, null); + byte[] keyBytes = zkClient.getData().forPath( + metastore_zk_path + "/keys/" + String.format(ZooKeeperTokenStore.ZK_SEQ_FORMAT, keySeq)); assertNotNull(keyBytes); assertEquals(new String(keyBytes), "key1Data"); @@ -116,8 +123,7 @@ public void testTokenStorage() throws Exception { HiveDelegationTokenSupport .encodeDelegationTokenInformation(tokenInfoRead)); - List allIds = ts - .getAllDelegationTokenIdentifiers(); + List allIds = ts.getAllDelegationTokenIdentifiers(); assertEquals(1, allIds.size()); Assert.assertEquals(TokenStoreDelegationTokenSecretManager .encodeWritable(tokenId), @@ -138,10 +144,10 @@ public void testAclNoAuth() throws Exception { ts = new ZooKeeperTokenStore(); try { ts.setConf(conf); + ts.init(null, ServerMode.METASTORE); fail("expected ACL exception"); } catch (DelegationTokenStore.TokenStoreException e) { - assertEquals(e.getCause().getClass(), - KeeperException.NoAuthException.class); + assertEquals(KeeperException.NoAuthException.class, e.getCause().getClass()); } } @@ -159,10 +165,10 @@ public void testAclInvalid() throws Exception { ts = new ZooKeeperTokenStore(); try { ts.setConf(conf); + ts.init(null, ServerMode.METASTORE); fail("expected ACL exception"); } catch (DelegationTokenStore.TokenStoreException e) { - assertEquals(e.getCause().getClass(), - KeeperException.InvalidACLException.class); + assertEquals(KeeperException.InvalidACLException.class, e.getCause().getClass()); } } @@ -171,10 +177,11 @@ public void testAclPositive() throws Exception { Configuration conf = createConf(ZK_PATH); conf.set( HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_ACL, - "world:anyone:cdrwa,ip:127.0.0.1:cdrwa"); + "ip:127.0.0.1:cdrwa,world:anyone:cdrwa"); ts = new ZooKeeperTokenStore(); ts.setConf(conf); - List acl = zkClient.getACL(ZK_PATH, new Stat()); + ts.init(null, ServerMode.METASTORE); + List acl = zkClient.getACL().forPath(ZK_PATH + ServerMode.METASTORE); assertEquals(2, acl.size()); } diff --git a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java index 6561743..8126cdf 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java @@ -34,11 +34,12 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hive.service.server.HiveServer2; +import org.apache.hive.jdbc.miniHS2.MiniHS2; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -48,15 +49,12 @@ * TestBeeLineWithArgs - executes tests of the command-line arguments to BeeLine * */ -//public class TestBeeLineWithArgs extends TestCase { public class TestBeeLineWithArgs { // Default location of HiveServer2 - final private static String JDBC_URL = BeeLine.BEELINE_DEFAULT_JDBC_URL + "localhost:10000"; private static final String tableName = "TestBeelineTable1"; private static final String tableComment = "Test table comment"; - - private static HiveServer2 hiveServer2; + private static MiniHS2 miniHS2; private List getBaseArgs(String jdbcUrl) { List argList = new ArrayList(8); @@ -74,17 +72,9 @@ public static void preTests() throws Exception { HiveConf hiveConf = new HiveConf(); // Set to non-zk lock manager to prevent HS2 from trying to connect hiveConf.setVar(HiveConf.ConfVars.HIVE_LOCK_MANAGER, "org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager"); - - // hiveConf.logVars(System.err); - // System.err.flush(); - - hiveServer2 = new HiveServer2(); - hiveServer2.init(hiveConf); - System.err.println("Starting HiveServer2..."); - hiveServer2.start(); - Thread.sleep(1000); + miniHS2 = new MiniHS2(hiveConf); + miniHS2.start(new HashMap()); createTable(); - } /** @@ -94,7 +84,7 @@ public static void preTests() throws Exception { */ private static void createTable() throws ClassNotFoundException, SQLException { Class.forName(BeeLine.BEELINE_DEFAULT_JDBC_DRIVER); - Connection con = DriverManager.getConnection(JDBC_URL,"", ""); + Connection con = DriverManager.getConnection(miniHS2.getBaseJdbcURL(),"", ""); assertNotNull("Connection is null", con); assertFalse("Connection should not be closed", con.isClosed()); @@ -129,13 +119,8 @@ private static void createTable() throws ClassNotFoundException, SQLException { */ @AfterClass public static void postTests() { - try { - if (hiveServer2 != null) { - System.err.println("Stopping HiveServer2..."); - hiveServer2.stop(); - } - } catch (Throwable t) { - t.printStackTrace(); + if (miniHS2.isStarted()) { + miniHS2.stop(); } } @@ -164,22 +149,20 @@ private String testCommandLineScript(List argList, InputStream inputStre * Test for presence of an expected pattern * in the output (stdout or stderr), fail if not found * Print PASSED or FAILED - * @paramm testName Name of test to print * @param expectedPattern Text to look for in command output/error * @param shouldMatch true if the pattern should be found, false if it should not * @throws Exception on command execution error */ - private void testScriptFile(String testName, String scriptText, String expectedPattern, + private void testScriptFile(String scriptText, String expectedPattern, boolean shouldMatch, List argList) throws Throwable { // Put the script content in a temp file - File scriptFile = File.createTempFile(testName, "temp"); + File scriptFile = File.createTempFile(this.getClass().getSimpleName(), "temp"); scriptFile.deleteOnExit(); PrintStream os = new PrintStream(new FileOutputStream(scriptFile)); os.print(scriptText); os.close(); - System.out.println(">>> STARTED -f " + testName); { List copy = new ArrayList(argList); copy.add("-f"); @@ -189,12 +172,11 @@ private void testScriptFile(String testName, String scriptText, String expectedP boolean matches = output.contains(expectedPattern); if (shouldMatch != matches) { //failed - fail(testName + ": Output" + output + " should" + (shouldMatch ? "" : " not") + + fail("Output" + output + " should" + (shouldMatch ? "" : " not") + " contain " + expectedPattern); } } - System.out.println(">>> STARTED -i " + testName); { List copy = new ArrayList(argList); copy.add("-i"); @@ -204,7 +186,7 @@ private void testScriptFile(String testName, String scriptText, String expectedP boolean matches = output.contains(expectedPattern); if (shouldMatch != matches) { //failed - fail(testName + ": Output" + output + " should" + (shouldMatch ? "" : " not") + + fail("Output" + output + " should" + (shouldMatch ? "" : " not") + " contain " + expectedPattern); } } @@ -217,11 +199,10 @@ private void testScriptFile(String testName, String scriptText, String expectedP */ @Test public void testWhitespaceBeforeCommentScriptFile() throws Throwable { - final String TEST_NAME = "testWhitespaceBeforeCommentScriptFile"; - final String SCRIPT_TEXT = " -- comment has spaces and tabs before it\n # comment has spaces and tabs before it\n"; - final String EXPECTED_PATTERN = "cannot recognize input near ''"; - List argList = getBaseArgs(JDBC_URL); - testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, false, argList); + final String SCRIPT_TEXT = " -- comment has spaces and tabs before it\n # comment has spaces and tabs before it\n"; + final String EXPECTED_PATTERN = "cannot recognize input near ''"; + List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, false, argList); } /** @@ -232,11 +213,10 @@ public void testWhitespaceBeforeCommentScriptFile() throws Throwable { */ @Test public void testPositiveScriptFile() throws Throwable { - final String TEST_NAME = "testPositiveScriptFile"; final String SCRIPT_TEXT = "show databases;\n"; final String EXPECTED_PATTERN = " default "; - List argList = getBaseArgs(JDBC_URL); - testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); + testScriptFile( SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); } /** @@ -247,24 +227,22 @@ public void testPositiveScriptFile() throws Throwable { */ @Test public void testBeelineHiveVariable() throws Throwable { - List argList = getBaseArgs(JDBC_URL); + List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--hivevar"); argList.add("DUMMY_TBL=dummy"); - final String TEST_NAME = "testHiveCommandLineHiveVariable"; final String SCRIPT_TEXT = "create table ${DUMMY_TBL} (d int);\nshow tables;\n"; final String EXPECTED_PATTERN = "dummy"; - testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); } @Test public void testBeelineHiveConfVariable() throws Throwable { - List argList = getBaseArgs(JDBC_URL); + List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--hiveconf"); - argList.add("hive.table.name=dummy"); - final String TEST_NAME = "testBeelineHiveConfVariable"; - final String SCRIPT_TEXT = "create table ${hiveconf:hive.table.name} (d int);\nshow tables;\n"; + argList.add("test.hive.table.name=dummy"); + final String SCRIPT_TEXT = "create table ${hiveconf:test.hive.table.name} (d int);\nshow tables;\n"; final String EXPECTED_PATTERN = "dummy"; - testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); } /** @@ -274,7 +252,7 @@ public void testBeelineHiveConfVariable() throws Throwable { */ @Test public void testBeelineMultiHiveVariable() throws Throwable { - List argList = getBaseArgs(JDBC_URL); + List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--hivevar"); argList.add("TABLE_NAME=dummy2"); @@ -289,10 +267,9 @@ public void testBeelineMultiHiveVariable() throws Throwable { argList.add("--hiveconf"); argList.add("COLUMN_TYPE=int"); - final String TEST_NAME = "testHiveCommandLineHiveVariable"; final String SCRIPT_TEXT = "${COMMAND} ${OBJECT} ${TABLE_NAME} (${hiveconf:COLUMN_NAME} ${hiveconf:COLUMN_TYPE});\nshow tables;\n"; final String EXPECTED_PATTERN = "dummy2"; - testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); } /** @@ -302,20 +279,18 @@ public void testBeelineMultiHiveVariable() throws Throwable { */ @Test public void testBreakOnErrorScriptFile() throws Throwable { - List argList = getBaseArgs(JDBC_URL); - final String TEST_NAME = "testBreakOnErrorScriptFile"; + List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); final String SCRIPT_TEXT = "select * from abcdefg01;\nshow databases;\n"; final String EXPECTED_PATTERN = " default "; - testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, false, argList); + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, false, argList); } @Test public void testBeelineShellCommand() throws Throwable { - List argList = getBaseArgs(JDBC_URL); - final String TEST_NAME = "testBeelineShellCommand"; + List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); final String SCRIPT_TEXT = "!sh echo \"hello world.\" > hw.txt\n!sh cat hw.txt\n!rm hw.txt"; final String EXPECTED_PATTERN = "hello world"; - testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); } /** @@ -324,11 +299,10 @@ public void testBeelineShellCommand() throws Throwable { */ @Test public void testNullDefault() throws Throwable { - final String TEST_NAME = "testNullDefault"; final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + "select null from " + tableName + " limit 1 ;\n"; final String EXPECTED_PATTERN = "NULL"; - testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(JDBC_URL)); + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(miniHS2.getBaseJdbcURL())); } /** @@ -337,39 +311,123 @@ public void testNullDefault() throws Throwable { */ @Test public void testNullNonEmpty() throws Throwable { - final String TEST_NAME = "testNullNonDefault"; final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + "!set nullemptystring false\n select null from " + tableName + " limit 1 ;\n"; final String EXPECTED_PATTERN = "NULL"; - testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(JDBC_URL)); + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(miniHS2.getBaseJdbcURL())); } @Test public void testGetVariableValue() throws Throwable { - final String TEST_NAME = "testGetVariableValue"; final String SCRIPT_TEXT = "set env:TERM;"; final String EXPECTED_PATTERN = "env:TERM"; - testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(JDBC_URL)); + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(miniHS2.getBaseJdbcURL())); } /** * Select null from table , check if setting null to empty string works. - * Original beeline/sqlline used to print nulls as empty strings + * Original beeline/sqlline used to print nulls as empty strings. + * Also test csv2 output format * Print PASSED or FAILED */ @Test public void testNullEmpty() throws Throwable { - final String TEST_NAME = "testNullNonDefault"; final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + "!set nullemptystring true\n select 'abc',null,'def' from " + tableName + " limit 1 ;\n"; final String EXPECTED_PATTERN = "abc,,def"; - List argList = getBaseArgs(JDBC_URL); + List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); + argList.add("--outputformat=csv2"); + + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + } + + /** + * Test writing output using DSV format, with custom delimiter ";" + */ + @Test + public void testDSVOutput() throws Throwable { + String SCRIPT_TEXT = getFormatTestQuery(); + List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); + argList.add("--outputformat=dsv"); + argList.add("--delimiterForDSV=;"); + + final String EXPECTED_PATTERN = "1;NULL;defg;\"ab\"\"c\";1.0"; + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + } + + /** + * Test writing output using TSV (new) format + */ + @Test + public void testTSV2Output() throws Throwable { + String SCRIPT_TEXT = getFormatTestQuery(); + List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); + argList.add("--outputformat=tsv2"); + + final String EXPECTED_PATTERN = "1\tNULL\tdefg\t\"ab\"\"c\"\t1.0"; + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + } + + /** + * Test writing output using TSV deprecated format + */ + @Test + public void testTSVOutput() throws Throwable { + String SCRIPT_TEXT = getFormatTestQuery(); + List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); + argList.add("--outputformat=tsv"); + + final String EXPECTED_PATTERN = "'1'\t'NULL'\t'defg'\t'ab\"c\'\t'1.0'"; + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + } + + + /** + * Test writing output using TSV deprecated format + * Check for deprecation message + */ + @Test + public void testTSVOutputDeprecation() throws Throwable { + String SCRIPT_TEXT = getFormatTestQuery(); + List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); + argList.add("--outputformat=tsv"); + + final String EXPECTED_PATTERN = "Format tsv is deprecated, please use tsv2"; + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + } + + /** + * Test writing output using CSV deprecated format + * Check for deprecation message + */ + @Test + public void testCSVOutputDeprecation() throws Throwable { + String SCRIPT_TEXT = getFormatTestQuery(); + List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--outputformat=csv"); - testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + final String EXPECTED_PATTERN = "Format csv is deprecated, please use csv2"; + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + } + + /** + * Test writing output using CSV deprecated format + */ + @Test + public void testCSVOutput() throws Throwable { + String SCRIPT_TEXT = getFormatTestQuery(); + List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); + argList.add("--outputformat=csv"); + final String EXPECTED_PATTERN = "'1','NULL','defg','ab\"c\','1.0'"; + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); } + + private String getFormatTestQuery() { + return "set hive.support.concurrency = false;\n" + + "select 1, null, 'defg', 'ab\"c', 1.0D from " + tableName + " limit 1 ;\n"; + } /** * Select null from table , check if setting null to empty string works - Using beeling cmd line * argument. @@ -378,49 +436,36 @@ public void testNullEmpty() throws Throwable { */ @Test public void testNullEmptyCmdArg() throws Throwable { - final String TEST_NAME = "testNullNonDefault"; final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + "select 'abc',null,'def' from " + tableName + " limit 1 ;\n"; - //final String EXPECTED_PATTERN = "| abc | | def |"; - final String EXPECTED_PATTERN = "abc,,def"; + final String EXPECTED_PATTERN = "'abc','','def'"; - List argList = getBaseArgs(JDBC_URL); + List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--nullemptystring=true"); argList.add("--outputformat=csv"); - testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); } /** * Attempt to execute a missing script file with the -f option to BeeLine - * Print PASSED or FAILED */ @Test public void testNegativeScriptFile() throws Throwable { - final String TEST_NAME = "testNegativeScriptFile"; final String EXPECTED_PATTERN = " default "; - long startTime = System.currentTimeMillis(); - System.out.println(">>> STARTED " + TEST_NAME); - // Create and delete a temp file File scriptFile = File.createTempFile("beelinenegative", "temp"); scriptFile.delete(); - List argList = getBaseArgs(JDBC_URL); + List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("-f"); argList.add(scriptFile.getAbsolutePath()); try { - String output = testCommandLineScript(argList, null); - long elapsedTime = (System.currentTimeMillis() - startTime)/1000; - String time = "(" + elapsedTime + "s)"; + String output = testCommandLineScript(argList, null); if (output.contains(EXPECTED_PATTERN)) { - System.err.println("Output: " + output); - System.err.println(">>> FAILED " + TEST_NAME + " (ERROR) " + time); - fail(TEST_NAME); - } else { - System.out.println(">>> PASSED " + TEST_NAME + " " + time); + fail("Output: " + output + " Negative pattern: " + EXPECTED_PATTERN); } } catch (Throwable e) { e.printStackTrace(); @@ -456,11 +501,10 @@ public void testNPE() throws UnsupportedEncodingException { @Test public void testHiveVarSubstitution() throws Throwable { - List argList = getBaseArgs(JDBC_URL + "#D_TBL=dummy_t"); - final String TEST_NAME = "testHiveVarSubstitution"; + List argList = getBaseArgs(miniHS2.getBaseJdbcURL() + "#D_TBL=dummy_t"); final String SCRIPT_TEXT = "create table ${D_TBL} (d int);\nshow tables;\n"; final String EXPECTED_PATTERN = "dummy_t"; - testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); } @Test @@ -469,13 +513,12 @@ public void testEmbeddedBeelineConnection() throws Throwable{ List argList = getBaseArgs(embeddedJdbcURL); argList.add("--hivevar"); argList.add("DUMMY_TBL=embedded_table"); - final String TEST_NAME = "testEmbeddedBeelineConnection"; // Set to non-zk lock manager to avoid trying to connect to zookeeper final String SCRIPT_TEXT = "set hive.lock.manager=org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager;\n" + "create table ${DUMMY_TBL} (d int);\nshow tables;\n"; final String EXPECTED_PATTERN = "embedded_table"; - testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); } /** @@ -484,11 +527,10 @@ public void testEmbeddedBeelineConnection() throws Throwable{ */ @Test public void testQueryProgress() throws Throwable { - final String TEST_NAME = "testQueryProgress"; final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + "select count(*) from " + tableName + ";\n"; final String EXPECTED_PATTERN = "Parsing command"; - testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(JDBC_URL)); + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(miniHS2.getBaseJdbcURL())); } /** @@ -497,11 +539,10 @@ public void testQueryProgress() throws Throwable { */ @Test public void testQueryProgressHidden() throws Throwable { - final String TEST_NAME = "testQueryProgress"; final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + "!set silent true\n" + "select count(*) from " + tableName + ";\n"; final String EXPECTED_PATTERN = "Parsing command"; - testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, false, getBaseArgs(JDBC_URL)); + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, false, getBaseArgs(miniHS2.getBaseJdbcURL())); } } diff --git a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java index 749fb9e..9ae9bc0 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java @@ -19,9 +19,12 @@ package org.apache.hive.beeline; import java.io.BufferedWriter; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileWriter; import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; import java.util.Random; import junit.framework.TestCase; @@ -31,14 +34,14 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaException; import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfo; -import org.apache.hive.beeline.HiveSchemaHelper; import org.apache.hive.beeline.HiveSchemaHelper.NestedScriptParser; -import org.apache.hive.beeline.HiveSchemaTool; public class TestSchemaTool extends TestCase { private HiveSchemaTool schemaTool; private HiveConf hiveConf; private String testMetastoreDB; + private PrintStream errStream; + private PrintStream outStream; @Override protected void setUp() throws Exception { @@ -48,8 +51,11 @@ protected void setUp() throws Exception { System.setProperty(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname, "jdbc:derby:" + testMetastoreDB + ";create=true"); hiveConf = new HiveConf(this.getClass()); - schemaTool = new HiveSchemaTool(System.getProperty("test.tmp.dir"), hiveConf, "derby"); + schemaTool = new HiveSchemaTool( + System.getProperty("test.tmp.dir", "target/tmp"), hiveConf, "derby"); System.setProperty("beeLine.system.exit", "true"); + errStream = System.err; + outStream = System.out; } @Override @@ -58,6 +64,8 @@ protected void tearDown() throws Exception { if (metaStoreDir.exists()) { FileUtils.forceDeleteOnExit(metaStoreDir); } + System.setOut(outStream); + System.setErr(errStream); } /** @@ -121,12 +129,42 @@ public void testSchemaUpgrade() throws Exception { foundException = true; } if (!foundException) { - throw new Exception("Hive operations shouldn't pass with older version schema"); + throw new Exception( + "Hive operations shouldn't pass with older version schema"); } - // upgrade schema from 0.7.0 to latest + // Generate dummy pre-upgrade script with errors + String invalidPreUpgradeScript = writeDummyPreUpgradeScript( + 0, "upgrade-0.11.0-to-0.12.0.derby.sql", "foo bar;"); + // Generate dummy pre-upgrade scripts with valid SQL + String validPreUpgradeScript0 = writeDummyPreUpgradeScript( + 0, "upgrade-0.12.0-to-0.13.0.derby.sql", + "CREATE TABLE schema_test0 (id integer);"); + String validPreUpgradeScript1 = writeDummyPreUpgradeScript( + 1, "upgrade-0.12.0-to-0.13.0.derby.sql", + "CREATE TABLE schema_test1 (id integer);"); + + // Capture system out and err + schemaTool.setVerbose(true); + OutputStream stderr = new ByteArrayOutputStream(); + PrintStream errPrintStream = new PrintStream(stderr); + System.setErr(errPrintStream); + OutputStream stdout = new ByteArrayOutputStream(); + PrintStream outPrintStream = new PrintStream(stdout); + System.setOut(outPrintStream); + + // Upgrade schema from 0.7.0 to latest schemaTool.doUpgrade("0.7.0"); - // verify that driver works fine with latest schema + + // Verify that the schemaTool ran pre-upgrade scripts and ignored errors + assertTrue(stderr.toString().contains(invalidPreUpgradeScript)); + assertTrue(stderr.toString().contains("foo")); + assertFalse(stderr.toString().contains(validPreUpgradeScript0)); + assertFalse(stderr.toString().contains(validPreUpgradeScript1)); + assertTrue(stdout.toString().contains(validPreUpgradeScript0)); + assertTrue(stdout.toString().contains(validPreUpgradeScript1)); + + // Verify that driver works fine with latest schema schemaTool.verifySchemaVersion(); } @@ -152,9 +190,9 @@ public void testScripts() throws Exception { String expectedSQL = StringUtils.join(resultScript, System.getProperty("line.separator")) + System.getProperty("line.separator"); File testScriptFile = generateTestScript(testScript); - String flattenedSql = HiveSchemaTool.buildCommand( - HiveSchemaHelper.getDbCommandParser("derby"), - testScriptFile.getParentFile().getPath(), testScriptFile.getName()); + String flattenedSql = HiveSchemaHelper.getDbCommandParser("derby") + .buildCommand(testScriptFile.getParentFile().getPath(), + testScriptFile.getName()); assertEquals(expectedSQL, flattenedSql); } @@ -194,9 +232,9 @@ public void testNestedScriptsForDerby() throws Exception { }; File testScriptFile = generateTestScript(parentTestScript); - String flattenedSql = HiveSchemaTool.buildCommand( - HiveSchemaHelper.getDbCommandParser("derby"), - testScriptFile.getParentFile().getPath(), testScriptFile.getName()); + String flattenedSql = HiveSchemaHelper.getDbCommandParser("derby") + .buildCommand(testScriptFile.getParentFile().getPath(), + testScriptFile.getName()); assertFalse(flattenedSql.contains("RUN")); assertFalse(flattenedSql.contains("comment")); assertTrue(flattenedSql.contains(childTab1)); @@ -239,9 +277,9 @@ public void testNestedScriptsForMySQL() throws Exception { }; File testScriptFile = generateTestScript(parentTestScript); - String flattenedSql = HiveSchemaTool.buildCommand( - HiveSchemaHelper.getDbCommandParser("mysql"), - testScriptFile.getParentFile().getPath(), testScriptFile.getName()); + String flattenedSql = HiveSchemaHelper.getDbCommandParser("mysql") + .buildCommand(testScriptFile.getParentFile().getPath(), + testScriptFile.getName()); assertFalse(flattenedSql.contains("RUN")); assertFalse(flattenedSql.contains("comment")); assertTrue(flattenedSql.contains(childTab1)); @@ -282,8 +320,8 @@ public void testScriptWithDelimiter() throws Exception { System.getProperty("line.separator"); File testScriptFile = generateTestScript(testScript); NestedScriptParser testDbParser = HiveSchemaHelper.getDbCommandParser("mysql"); - String flattenedSql = HiveSchemaTool.buildCommand(testDbParser, - testScriptFile.getParentFile().getPath(), testScriptFile.getName()); + String flattenedSql = testDbParser.buildCommand(testScriptFile.getParentFile().getPath(), + testScriptFile.getName()); assertEquals(expectedSQL, flattenedSql); } @@ -317,8 +355,8 @@ public void testScriptMultiRowComment() throws Exception { System.getProperty("line.separator"); File testScriptFile = generateTestScript(testScript); NestedScriptParser testDbParser = HiveSchemaHelper.getDbCommandParser("mysql"); - String flattenedSql = HiveSchemaTool.buildCommand(testDbParser, - testScriptFile.getParentFile().getPath(), testScriptFile.getName()); + String flattenedSql = testDbParser.buildCommand(testScriptFile.getParentFile().getPath(), + testScriptFile.getName()); assertEquals(expectedSQL, flattenedSql); } @@ -358,9 +396,9 @@ public void testNestedScriptsForOracle() throws Exception { }; File testScriptFile = generateTestScript(parentTestScript); - String flattenedSql = HiveSchemaTool.buildCommand( - HiveSchemaHelper.getDbCommandParser("oracle"), - testScriptFile.getParentFile().getPath(), testScriptFile.getName()); + String flattenedSql = HiveSchemaHelper.getDbCommandParser("oracle") + .buildCommand(testScriptFile.getParentFile().getPath(), + testScriptFile.getName()); assertFalse(flattenedSql.contains("@")); assertFalse(flattenedSql.contains("comment")); assertTrue(flattenedSql.contains(childTab1)); @@ -380,4 +418,21 @@ private File generateTestScript(String [] stmts) throws IOException { out.close(); return testScriptFile; } -} + + /** + * Write out a dummy pre-upgrade script with given SQL statement. + */ + private String writeDummyPreUpgradeScript(int index, String upgradeScriptName, + String sql) throws Exception { + String preUpgradeScript = "pre-" + index + "-" + upgradeScriptName; + String dummyPreScriptPath = System.getProperty("test.tmp.dir", "target/tmp") + + File.separatorChar + "scripts" + File.separatorChar + "metastore" + + File.separatorChar + "upgrade" + File.separatorChar + "derby" + + File.separatorChar + preUpgradeScript; + FileWriter fstream = new FileWriter(dummyPreScriptPath); + BufferedWriter out = new BufferedWriter(fstream); + out.write(sql + System.getProperty("line.separator") + ";"); + out.close(); + return preUpgradeScript; + } +} \ No newline at end of file diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java index 15ff1f4..6a18b9a 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java @@ -39,6 +39,7 @@ import java.sql.Statement; import java.sql.Types; import java.util.ArrayList; +import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -417,50 +418,18 @@ public void testPrepareStatement() { /////////////////////////////////////////////// //////////////////// correct testcase + //////////////////// executed twice: once with the typed ps setters, once with the generic setObject ////////////////////////////////////////////// try { - PreparedStatement ps = con.prepareStatement(sql); - - ps.setBoolean(1, true); - ps.setBoolean(2, true); - - ps.setShort(3, Short.valueOf("1")); - ps.setInt(4, 2); - ps.setFloat(5, 3f); - ps.setDouble(6, Double.valueOf(4)); - ps.setString(7, "test'string\""); - ps.setLong(8, 5L); - ps.setByte(9, (byte) 1); - ps.setByte(10, (byte) 1); - ps.setString(11, "2012-01-01"); - - ps.setMaxRows(2); - - assertTrue(true); - + PreparedStatement ps = createPreapredStatementUsingSetXXX(sql); ResultSet res = ps.executeQuery(); - assertNotNull(res); - - while (res.next()) { - assertEquals("2011-03-25", res.getString("ddate")); - assertEquals("10", res.getString("num")); - assertEquals((byte) 10, res.getByte("num")); - assertEquals("2011-03-25", res.getDate("ddate").toString()); - assertEquals(Double.valueOf(10).doubleValue(), res.getDouble("num"), 0.1); - assertEquals(10, res.getInt("num")); - assertEquals(Short.valueOf("10").shortValue(), res.getShort("num")); - assertEquals(10L, res.getLong("num")); - assertEquals(true, res.getBoolean("bv")); - Object o = res.getObject("ddate"); - assertNotNull(o); - o = res.getObject("num"); - assertNotNull(o); - } - res.close(); - assertTrue(true); + assertPreparedStatementResultAsExpected(res); + ps.close(); + ps = createPreapredStatementUsingSetObject(sql); + res = ps.executeQuery(); + assertPreparedStatementResultAsExpected(res); ps.close(); - assertTrue(true); } catch (Exception e) { e.printStackTrace(); @@ -515,6 +484,82 @@ public void testPrepareStatement() { assertNotNull( "Execute the invalid setted sql statement should throw exception", expectedException); + + // setObject to the yet unknown type java.util.Date + expectedException = null; + try { + PreparedStatement ps = con.prepareStatement(sql); + ps.setObject(1, new Date()); + ps.executeQuery(); + } catch (Exception e) { + expectedException = e; + } + assertNotNull( + "Setting to an unknown type should throw an exception", + expectedException); + + } + + private PreparedStatement createPreapredStatementUsingSetObject(String sql) throws SQLException { + PreparedStatement ps = con.prepareStatement(sql); + + ps.setObject(1, true); //setBoolean + ps.setObject(2, true); //setBoolean + + ps.setObject(3, Short.valueOf("1")); //setShort + ps.setObject(4, 2); //setInt + ps.setObject(5, 3f); //setFloat + ps.setObject(6, Double.valueOf(4)); //setDouble + ps.setObject(7, "test'string\""); //setString + ps.setObject(8, 5L); //setLong + ps.setObject(9, (byte) 1); //setByte + ps.setObject(10, (byte) 1); //setByte + ps.setString(11, "2012-01-01"); //setString + + ps.setMaxRows(2); + return ps; + } + + private PreparedStatement createPreapredStatementUsingSetXXX(String sql) throws SQLException { + PreparedStatement ps = con.prepareStatement(sql); + + ps.setBoolean(1, true); //setBoolean + ps.setBoolean(2, true); //setBoolean + + ps.setShort(3, Short.valueOf("1")); //setShort + ps.setInt(4, 2); //setInt + ps.setFloat(5, 3f); //setFloat + ps.setDouble(6, Double.valueOf(4)); //setDouble + ps.setString(7, "test'string\""); //setString + ps.setLong(8, 5L); //setLong + ps.setByte(9, (byte) 1); //setByte + ps.setByte(10, (byte) 1); //setByte + ps.setString(11, "2012-01-01"); //setString + + ps.setMaxRows(2); + return ps; + } + + private void assertPreparedStatementResultAsExpected(ResultSet res ) throws SQLException { + assertNotNull(res); + + while (res.next()) { + assertEquals("2011-03-25", res.getString("ddate")); + assertEquals("10", res.getString("num")); + assertEquals((byte) 10, res.getByte("num")); + assertEquals("2011-03-25", res.getDate("ddate").toString()); + assertEquals(Double.valueOf(10).doubleValue(), res.getDouble("num"), 0.1); + assertEquals(10, res.getInt("num")); + assertEquals(Short.valueOf("10").shortValue(), res.getShort("num")); + assertEquals(10L, res.getLong("num")); + assertEquals(true, res.getBoolean("bv")); + Object o = res.getObject("ddate"); + assertNotNull(o); + o = res.getObject("num"); + assertNotNull(o); + } + res.close(); + assertTrue(true); } /** @@ -1682,29 +1727,24 @@ public void testDriverProperties() throws SQLException { } private static final String[][] HTTP_URL_PROPERTIES = new String[][] { - {"jdbc:hive2://server:10002/db;" + - "user=foo;password=bar?" + - "hive.server2.transport.mode=http;" + - "hive.server2.thrift.http.path=hs2", "server", "10002", "db", "http", "hs2"}, - {"jdbc:hive2://server:10000/testdb;" + - "user=foo;password=bar?" + - "hive.server2.transport.mode=binary;" + - "hive.server2.thrift.http.path=", "server", "10000", "testdb", "binary", ""}, - }; - - @Test - public void testParseUrlHttpMode() throws SQLException, JdbcUriParseException, - ZooKeeperHiveClientException { - new HiveDriver(); - for (String[] testValues : HTTP_URL_PROPERTIES) { - JdbcConnectionParams params = Utils.parseURL(testValues[0]); - assertEquals(params.getHost(), testValues[1]); - assertEquals(params.getPort(), Integer.parseInt(testValues[2])); - assertEquals(params.getDbName(), testValues[3]); - assertEquals(params.getHiveConfs().get("hive.server2.transport.mode"), testValues[4]); - assertEquals(params.getHiveConfs().get("hive.server2.thrift.http.path"), testValues[5]); - } + { "jdbc:hive2://server:10002/db;user=foo;password=bar;transportMode=http;httpPath=hs2", + "server", "10002", "db", "http", "hs2" }, + { "jdbc:hive2://server:10000/testdb;user=foo;password=bar;transportMode=binary;httpPath=", + "server", "10000", "testdb", "binary", "" }, }; + +@Test +public void testParseUrlHttpMode() throws SQLException, JdbcUriParseException, + ZooKeeperHiveClientException { + new HiveDriver(); + for (String[] testValues : HTTP_URL_PROPERTIES) { + JdbcConnectionParams params = Utils.parseURL(testValues[0]); + assertEquals(params.getHost(), testValues[1]); + assertEquals(params.getPort(), Integer.parseInt(testValues[2])); + assertEquals(params.getDbName(), testValues[3]); + assertEquals(params.getSessionVars().get("transportMode"), testValues[4]); + assertEquals(params.getSessionVars().get("httpPath"), testValues[5]); } +} private static void assertDpi(DriverPropertyInfo dpi, String name, String value) { diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestNoSaslAuth.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestNoSaslAuth.java new file mode 100644 index 0000000..84f1168 --- /dev/null +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestNoSaslAuth.java @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.jdbc; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.util.HashMap; +import java.util.Map; + +import junit.framework.Assert; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hive.jdbc.miniHS2.MiniHS2; +import org.apache.hive.service.cli.HiveSQLException; +import org.apache.hive.service.cli.session.HiveSessionHook; +import org.apache.hive.service.cli.session.HiveSessionHookContext; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestNoSaslAuth { + private static MiniHS2 miniHS2 = null; + private static String sessionUserName = ""; + + public static class NoSaslSessionHook implements HiveSessionHook { + public static boolean checkUser = false; + + @Override + public void run(HiveSessionHookContext sessionHookContext) + throws HiveSQLException { + if (checkUser) { + Assert.assertEquals(sessionHookContext.getSessionUser(), sessionUserName); + } + } + } + + private Connection hs2Conn = null; + + @BeforeClass + public static void beforeTest() throws Exception { + Class.forName(MiniHS2.getJdbcDriverName()); + HiveConf conf = new HiveConf(); + conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); + conf.setBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS, false); + conf.setVar(ConfVars.HIVE_SERVER2_SESSION_HOOK, + NoSaslSessionHook.class.getName()); + conf.setVar(ConfVars.HIVE_SERVER2_AUTHENTICATION, "NOSASL"); + miniHS2 = new MiniHS2(conf); + Map overlayProps = new HashMap(); + miniHS2.start(overlayProps); + } + + @Before + public void setUp() throws Exception { + // enable the hook check after the server startup, + NoSaslSessionHook.checkUser = true; + } + + @After + public void tearDown() throws Exception { + hs2Conn.close(); + NoSaslSessionHook.checkUser = false; + } + + @AfterClass + public static void afterTest() throws Exception { + if (miniHS2.isStarted()) + miniHS2.stop(); + } + + /** + * Initiate a non-sasl connection. The session hook will verfiy the user name + * set correctly + * + * @throws Exception + */ + @Test + public void testConnection() throws Exception { + sessionUserName = "user1"; + hs2Conn = DriverManager.getConnection( + miniHS2.getJdbcURL() + ";auth=noSasl", sessionUserName, "foo"); + } +} diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestSSL.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestSSL.java index 0058cd3..2bc8d73 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestSSL.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestSSL.java @@ -21,6 +21,7 @@ import static org.junit.Assert.fail; import java.io.File; +import java.net.URLEncoder; import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; @@ -55,8 +56,8 @@ private Connection hs2Conn = null; private String dataFileDir = conf.get("test.data.files"); private Map confOverlay; - private final String SSL_CONN_PARAMS = ";ssl=true;sslTrustStore=" + dataFileDir + File.separator + - TRUST_STORE_NAME + ";trustStorePassword=" + KEY_STORE_PASSWORD; + private final String SSL_CONN_PARAMS = ";ssl=true;sslTrustStore=" + URLEncoder.encode(dataFileDir + File.separator + + TRUST_STORE_NAME) + ";trustStorePassword=" + KEY_STORE_PASSWORD; @BeforeClass public static void beforeTest() throws Exception { diff --git a/itests/pom.xml b/itests/pom.xml index 33c67bf..a15e04a 100644 --- a/itests/pom.xml +++ b/itests/pom.xml @@ -19,7 +19,7 @@ org.apache.hive hive - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../pom.xml diff --git a/itests/qtest-spark/pom.xml b/itests/qtest-spark/pom.xml index 338cad3..b31db8e 100644 --- a/itests/qtest-spark/pom.xml +++ b/itests/qtest-spark/pom.xml @@ -20,7 +20,7 @@ org.apache.hive hive-it - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../pom.xml diff --git a/itests/qtest/pom.xml b/itests/qtest/pom.xml index 7a932ce..376f4a9 100644 --- a/itests/qtest/pom.xml +++ b/itests/qtest/pom.xml @@ -20,7 +20,7 @@ org.apache.hive hive-it - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../pom.xml diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index c429799..c178856 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -127,6 +127,7 @@ minitez.query.files.shared=alter_merge_2_orc.q,\ script_pipe.q,\ scriptfile1.q,\ select_dummy_source.q,\ + skewjoin.q,\ stats_counter.q,\ stats_counter_partitioned.q,\ stats_noscan_1.q,\ @@ -156,6 +157,7 @@ minitez.query.files.shared=alter_merge_2_orc.q,\ update_where_partitioned.q,\ update_two_cols.q,\ vector_between_in.q,\ + vector_bucket.q,\ vector_cast_constant.q,\ vector_char_4.q,\ vector_char_simple.q,\ @@ -165,10 +167,13 @@ minitez.query.files.shared=alter_merge_2_orc.q,\ vector_distinct_2.q,\ vector_elt.q,\ vector_groupby_3.q,\ + vector_groupby_reduce.q,\ vector_left_outer_join.q,\ vector_mapjoin_reduce.q,\ vector_non_string_partition.q,\ vector_orderby_5.q,\ + vector_partitioned_date_time.q,\ + vector_reduce_groupby_decimal.q,\ vector_string_concat.q,\ vector_varchar_4.q,\ vector_varchar_simple.q,\ @@ -243,6 +248,7 @@ minitez.query.files=bucket_map_join_tez1.q,\ tez_union_decimal.q,\ tez_union_group_by.q,\ tez_smb_main.q,\ + tez_smb_1.q,\ vectorized_dynamic_partition_pruning.q beeline.positive.exclude=add_part_exist.q,\ diff --git a/itests/test-serde/pom.xml b/itests/test-serde/pom.xml index 4432084..c0135fc 100644 --- a/itests/test-serde/pom.xml +++ b/itests/test-serde/pom.xml @@ -19,7 +19,7 @@ org.apache.hive hive-it - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../pom.xml diff --git a/itests/test-serde/src/main/java/org/apache/hadoop/hive/serde2/TestSerDe.java b/itests/test-serde/src/main/java/org/apache/hadoop/hive/serde2/TestSerDe.java index c32ead7..9f7a20a 100644 --- a/itests/test-serde/src/main/java/org/apache/hadoop/hive/serde2/TestSerDe.java +++ b/itests/test-serde/src/main/java/org/apache/hadoop/hive/serde2/TestSerDe.java @@ -27,6 +27,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.objectinspector.MetadataListStructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; @@ -44,10 +45,17 @@ * TestSerDe. * */ +@SerDeSpec(schemaProps = { + serdeConstants.LIST_COLUMNS, serdeConstants.LIST_COLUMN_TYPES, + TestSerDe.COLUMNS, TestSerDe.COLUMNS_COMMENTS, TestSerDe.DEFAULT_SERIALIZATION_FORMAT}) public class TestSerDe extends AbstractSerDe { public static final Log LOG = LogFactory.getLog(TestSerDe.class.getName()); + public static final String COLUMNS = "columns"; + public static final String COLUMNS_COMMENTS = "columns.comments"; + public static final String DEFAULT_SERIALIZATION_FORMAT = "testserde.default.serialization.format"; + public String getShortName() { return shortName(); } @@ -76,7 +84,7 @@ public TestSerDe() throws SerDeException { @Override public void initialize(Configuration job, Properties tbl) throws SerDeException { separator = DefaultSeparator; - String altSep = tbl.getProperty("testserde.default.serialization.format"); + String altSep = tbl.getProperty(DEFAULT_SERIALIZATION_FORMAT); if (altSep != null && altSep.length() > 0) { try { byte[] b = new byte[1]; @@ -87,7 +95,7 @@ public void initialize(Configuration job, Properties tbl) throws SerDeException } } - String columnProperty = tbl.getProperty("columns"); + String columnProperty = tbl.getProperty(COLUMNS); if (columnProperty == null || columnProperty.length() == 0) { // Hack for tables with no columns // Treat it as a table with a single column called "col" @@ -97,7 +105,7 @@ public void initialize(Configuration job, Properties tbl) throws SerDeException } else { columnNames = Arrays.asList(columnProperty.split(",")); cachedObjectInspector = MetadataListStructObjectInspector - .getInstance(columnNames,Lists.newArrayList(Splitter.on('\0').split(tbl.getProperty("columns.comments")))); + .getInstance(columnNames,Lists.newArrayList(Splitter.on('\0').split(tbl.getProperty(COLUMNS_COMMENTS)))); } LOG.info(getClass().getName() + ": initialized with columnNames: " + columnNames); diff --git a/itests/util/pom.xml b/itests/util/pom.xml index 8a7f3a7..891f391 100644 --- a/itests/util/pom.xml +++ b/itests/util/pom.xml @@ -19,7 +19,7 @@ org.apache.hive hive-it - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../pom.xml diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index f5e35b8..ccaec46 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -305,7 +305,7 @@ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, this.outDir = outDir; this.logDir = logDir; if (confDir != null && !confDir.isEmpty()) { - HiveConf.setHiveSiteLocation(new URL("file://"+confDir+"/hive-site.xml")); + HiveConf.setHiveSiteLocation(new URL("file://"+ new File(confDir).toURI().getPath() + "/hive-site.xml")); System.out.println("Setting hive-site: "+HiveConf.getHiveSiteLocation()); } conf = new HiveConf(Driver.class); @@ -559,7 +559,7 @@ public void clearTestSideEffects() throws Exception { List indexes = db.getIndexes(dbName, tblName, (short)-1); if (indexes != null && indexes.size() > 0) { for (Index index : indexes) { - db.dropIndex(dbName, tblName, index.getIndexName(), true); + db.dropIndex(dbName, tblName, index.getIndexName(), true, true); } } } diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAccessControllerForTest.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAccessControllerForTest.java index 88004c8..25a8838 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAccessControllerForTest.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAccessControllerForTest.java @@ -39,12 +39,11 @@ @Override - public void applyAuthorizationConfigPolicy(HiveConf hiveConf) { + public void applyAuthorizationConfigPolicy(HiveConf hiveConf) throws HiveAuthzPluginException { super.applyAuthorizationConfigPolicy(hiveConf); // remove restrictions on the variables that can be set using set command - hiveConf.setIsModWhiteListEnabled(false); - + hiveConf.setModifiableWhiteListRegex(".*"); } } diff --git a/jdbc/pom.xml b/jdbc/pom.xml index 215dc86..9a569d1 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -19,7 +19,7 @@ org.apache.hive hive - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../pom.xml diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java index e0d2d6d..d71bde3 100644 --- a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java +++ b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java @@ -218,7 +218,7 @@ private String getServerHttpUrl(boolean useSsl) { String schemeName = useSsl ? "https" : "http"; // http path should begin with "/" String httpPath; - httpPath = hiveConfMap.get(JdbcConnectionParams.HTTP_PATH); + httpPath = sessConfMap.get(JdbcConnectionParams.HTTP_PATH); if (httpPath == null) { httpPath = "/"; } else if (!httpPath.startsWith("/")) { @@ -258,15 +258,12 @@ private DefaultHttpClient getHttpClient(Boolean useSsl) throws SQLException { HttpRequestInterceptor requestInterceptor; // If Kerberos if (isKerberosAuthMode()) { - if (useSsl) { - String msg = "SSL encryption is currently not supported with " + - "kerberos authentication"; - throw new SQLException(msg, " 08S01"); - } /** * Add an interceptor which sets the appropriate header in the request. * It does the kerberos authentication and get the final service ticket, * for sending to the server before every request. + * In https mode, the entire information is encrypted + * TODO: Optimize this with a mix of kerberos + using cookie. */ requestInterceptor = new HttpKerberosRequestInterceptor( sessConfMap.get(JdbcConnectionParams.AUTH_PRINCIPAL), host, getServerHttpUrl(false)); @@ -277,46 +274,46 @@ private DefaultHttpClient getHttpClient(Boolean useSsl) throws SQLException { * In https mode, the entire information is encrypted */ requestInterceptor = new HttpBasicAuthInterceptor(getUserName(), getPassword()); - // Configure httpClient for SSL - if (useSsl) { - String sslTrustStorePath = sessConfMap.get(JdbcConnectionParams.SSL_TRUST_STORE); - String sslTrustStorePassword = sessConfMap.get( - JdbcConnectionParams.SSL_TRUST_STORE_PASSWORD); - KeyStore sslTrustStore; - SSLSocketFactory socketFactory; - /** - * The code within the try block throws: - * 1. SSLInitializationException - * 2. KeyStoreException - * 3. IOException - * 4. NoSuchAlgorithmException - * 5. CertificateException - * 6. KeyManagementException - * 7. UnrecoverableKeyException - * We don't want the client to retry on any of these, hence we catch all - * and throw a SQLException. - */ - try { - if (sslTrustStorePath == null || sslTrustStorePath.isEmpty()) { - // Create a default socket factory based on standard JSSE trust material - socketFactory = SSLSocketFactory.getSocketFactory(); - } - else { - // Pick trust store config from the given path - sslTrustStore = KeyStore.getInstance(JdbcConnectionParams.SSL_TRUST_STORE_TYPE); - sslTrustStore.load(new FileInputStream(sslTrustStorePath), - sslTrustStorePassword.toCharArray()); - socketFactory = new SSLSocketFactory(sslTrustStore); - } - socketFactory.setHostnameVerifier(SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER); - Scheme sslScheme = new Scheme("https", 443, socketFactory); - httpClient.getConnectionManager().getSchemeRegistry().register(sslScheme); + } + // Configure httpClient for SSL + if (useSsl) { + String sslTrustStorePath = sessConfMap.get(JdbcConnectionParams.SSL_TRUST_STORE); + String sslTrustStorePassword = sessConfMap.get( + JdbcConnectionParams.SSL_TRUST_STORE_PASSWORD); + KeyStore sslTrustStore; + SSLSocketFactory socketFactory; + /** + * The code within the try block throws: + * 1. SSLInitializationException + * 2. KeyStoreException + * 3. IOException + * 4. NoSuchAlgorithmException + * 5. CertificateException + * 6. KeyManagementException + * 7. UnrecoverableKeyException + * We don't want the client to retry on any of these, hence we catch all + * and throw a SQLException. + */ + try { + if (sslTrustStorePath == null || sslTrustStorePath.isEmpty()) { + // Create a default socket factory based on standard JSSE trust material + socketFactory = SSLSocketFactory.getSocketFactory(); } - catch (Exception e) { - String msg = "Could not create an https connection to " + - jdbcUriString + ". " + e.getMessage(); - throw new SQLException(msg, " 08S01", e); + else { + // Pick trust store config from the given path + sslTrustStore = KeyStore.getInstance(JdbcConnectionParams.SSL_TRUST_STORE_TYPE); + sslTrustStore.load(new FileInputStream(sslTrustStorePath), + sslTrustStorePassword.toCharArray()); + socketFactory = new SSLSocketFactory(sslTrustStore); } + socketFactory.setHostnameVerifier(SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER); + Scheme sslScheme = new Scheme("https", 443, socketFactory); + httpClient.getConnectionManager().getSchemeRegistry().register(sslScheme); + } + catch (Exception e) { + String msg = "Could not create an https connection to " + + jdbcUriString + ". " + e.getMessage(); + throw new SQLException(msg, " 08S01", e); } } httpClient.addRequestInterceptor(requestInterceptor); @@ -443,6 +440,12 @@ private void openSession() throws SQLException { } openReq.setConfiguration(openConf); + // Store the user name in the open request in case no non-sasl authentication + if (JdbcConnectionParams.AUTH_SIMPLE.equals(sessConfMap.get(JdbcConnectionParams.AUTH_TYPE))) { + openReq.setUsername(sessConfMap.get(JdbcConnectionParams.AUTH_USER)); + openReq.setPassword(sessConfMap.get(JdbcConnectionParams.AUTH_PASSWD)); + } + try { TOpenSessionResp openResp = client.OpenSession(openReq); @@ -485,7 +488,7 @@ private boolean isKerberosAuthMode() { } private boolean isHttpTransportMode() { - String transportMode = hiveConfMap.get(JdbcConnectionParams.TRANSPORT_MODE); + String transportMode = sessConfMap.get(JdbcConnectionParams.TRANSPORT_MODE); if(transportMode != null && (transportMode.equalsIgnoreCase("http"))) { return true; } diff --git a/jdbc/src/java/org/apache/hive/jdbc/HivePreparedStatement.java b/jdbc/src/java/org/apache/hive/jdbc/HivePreparedStatement.java index 8ea125a..8a0671f 100644 --- a/jdbc/src/java/org/apache/hive/jdbc/HivePreparedStatement.java +++ b/jdbc/src/java/org/apache/hive/jdbc/HivePreparedStatement.java @@ -37,6 +37,8 @@ import java.sql.SQLXML; import java.sql.Time; import java.sql.Timestamp; +import java.sql.Types; +import java.text.MessageFormat; import java.util.Calendar; import java.util.HashMap; import java.util.Scanner; @@ -564,8 +566,7 @@ public void setNString(int parameterIndex, String value) throws SQLException { */ public void setNull(int parameterIndex, int sqlType) throws SQLException { - // TODO Auto-generated method stub - throw new SQLException("Method not supported"); + this.parameters.put(parameterIndex, "NULL"); } /* @@ -575,8 +576,7 @@ public void setNull(int parameterIndex, int sqlType) throws SQLException { */ public void setNull(int paramIndex, int sqlType, String typeName) throws SQLException { - // TODO Auto-generated method stub - throw new SQLException("Method not supported"); + this.parameters.put(paramIndex, "NULL"); } /* @@ -586,8 +586,38 @@ public void setNull(int paramIndex, int sqlType, String typeName) throws SQLExce */ public void setObject(int parameterIndex, Object x) throws SQLException { - // TODO Auto-generated method stub - throw new SQLException("Method not supported"); + if (x == null) { + setNull(parameterIndex, Types.NULL); + } else if (x instanceof String) { + setString(parameterIndex, (String) x); + } else if (x instanceof Short) { + setShort(parameterIndex, ((Short) x).shortValue()); + } else if (x instanceof Integer) { + setInt(parameterIndex, ((Integer) x).intValue()); + } else if (x instanceof Long) { + setLong(parameterIndex, ((Long) x).longValue()); + } else if (x instanceof Float) { + setFloat(parameterIndex, ((Float) x).floatValue()); + } else if (x instanceof Double) { + setDouble(parameterIndex, ((Double) x).doubleValue()); + } else if (x instanceof Boolean) { + setBoolean(parameterIndex, ((Boolean) x).booleanValue()); + } else if (x instanceof Byte) { + setByte(parameterIndex, ((Byte) x).byteValue()); + } else if (x instanceof Character) { + setString(parameterIndex, x.toString()); + } else if (x instanceof Timestamp) { + setString(parameterIndex, x.toString()); + } else if (x instanceof BigDecimal) { + setString(parameterIndex, x.toString()); + } else { + // Can't infer a type. + throw new SQLException( + MessageFormat + .format( + "Can''t infer the SQL type to use for an instance of {0}. Use setObject() with an explicit Types value to specify the type to use.", + x.getClass().getName())); + } } /* diff --git a/jdbc/src/java/org/apache/hive/jdbc/Utils.java b/jdbc/src/java/org/apache/hive/jdbc/Utils.java index e6b1a36..d53cde7 100644 --- a/jdbc/src/java/org/apache/hive/jdbc/Utils.java +++ b/jdbc/src/java/org/apache/hive/jdbc/Utils.java @@ -22,6 +22,7 @@ import java.net.URISyntaxException; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -65,7 +66,9 @@ // Client param names: static final String AUTH_TYPE = "auth"; - static final String AUTH_QOP = "sasl.qop"; + // We're deprecating this variable's name. + static final String AUTH_QOP_DEPRECATED = "sasl.qop"; + static final String AUTH_QOP = "saslQop"; static final String AUTH_SIMPLE = "noSasl"; static final String AUTH_TOKEN = "delegationToken"; static final String AUTH_USER = "user"; @@ -78,14 +81,23 @@ static final String USE_SSL = "ssl"; static final String SSL_TRUST_STORE = "sslTrustStore"; static final String SSL_TRUST_STORE_PASSWORD = "trustStorePassword"; - static final String TRANSPORT_MODE = "hive.server2.transport.mode"; - static final String HTTP_PATH = "hive.server2.thrift.http.path"; + // We're deprecating the name and placement of this in the parsed map (from hive conf vars to + // hive session vars). + static final String TRANSPORT_MODE_DEPRECATED = "hive.server2.transport.mode"; + static final String TRANSPORT_MODE = "transportMode"; + // We're deprecating the name and placement of this in the parsed map (from hive conf vars to + // hive session vars). + static final String HTTP_PATH_DEPRECATED = "hive.server2.thrift.http.path"; + static final String HTTP_PATH = "httpPath"; static final String SERVICE_DISCOVERY_MODE = "serviceDiscoveryMode"; - // Don't use dynamic serice discovery + // Don't use dynamic service discovery static final String SERVICE_DISCOVERY_MODE_NONE = "none"; // Use ZooKeeper for indirection while using dynamic service discovery static final String SERVICE_DISCOVERY_MODE_ZOOKEEPER = "zooKeeper"; static final String ZOOKEEPER_NAMESPACE = "zooKeeperNamespace"; + // Default namespace value on ZooKeeper. + // This value is used if the param "zooKeeperNamespace" is not specified in the JDBC Uri. + static final String ZOOKEEPER_DEFAULT_NAMESPACE = "hiveserver2"; // Non-configurable params: // ZOOKEEPER_SESSION_TIMEOUT is not exposed as client configurable @@ -214,10 +226,11 @@ public static void verifySuccess(TStatus status) throws SQLException { // Verify success and optionally with_info status, else throw SQLException public static void verifySuccess(TStatus status, boolean withInfo) throws SQLException { - if ((status.getStatusCode() != TStatusCode.SUCCESS_STATUS) && - (withInfo && (status.getStatusCode() != TStatusCode.SUCCESS_WITH_INFO_STATUS))) { - throw new HiveSQLException(status); + if (status.getStatusCode() == TStatusCode.SUCCESS_STATUS || + (withInfo && status.getStatusCode() == TStatusCode.SUCCESS_WITH_INFO_STATUS)) { + return; } + throw new HiveSQLException(status); } /** @@ -329,6 +342,25 @@ public static JdbcConnectionParams parseURL(String uri) throws JdbcUriParseExcep } } + // Handle all deprecations here: + String newUsage; + String usageUrlBase = "jdbc:hive2://:/dbName;"; + // Handle deprecation of AUTH_QOP_DEPRECATED + newUsage = usageUrlBase + JdbcConnectionParams.AUTH_QOP + "="; + handleParamDeprecation(connParams.getSessionVars(), connParams.getSessionVars(), + JdbcConnectionParams.AUTH_QOP_DEPRECATED, JdbcConnectionParams.AUTH_QOP, newUsage); + + // Handle deprecation of TRANSPORT_MODE_DEPRECATED + newUsage = usageUrlBase + JdbcConnectionParams.TRANSPORT_MODE + "="; + handleParamDeprecation(connParams.getHiveConfs(), connParams.getSessionVars(), + JdbcConnectionParams.TRANSPORT_MODE_DEPRECATED, JdbcConnectionParams.TRANSPORT_MODE, + newUsage); + + // Handle deprecation of HTTP_PATH_DEPRECATED + newUsage = usageUrlBase + JdbcConnectionParams.HTTP_PATH + "="; + handleParamDeprecation(connParams.getHiveConfs(), connParams.getSessionVars(), + JdbcConnectionParams.HTTP_PATH_DEPRECATED, JdbcConnectionParams.HTTP_PATH, newUsage); + // Extract host, port if (connParams.isEmbeddedMode()) { // In case of embedded mode we were supplied with an empty authority. @@ -339,6 +371,7 @@ public static JdbcConnectionParams parseURL(String uri) throws JdbcUriParseExcep // Else substitute the dummy authority with a resolved one. // In case of dynamic service discovery using ZooKeeper, it picks a server uri from ZooKeeper String resolvedAuthorityString = resolveAuthority(connParams); + LOG.info("Resolved authority: " + resolvedAuthorityString); uri = uri.replace(dummyAuthorityString, resolvedAuthorityString); connParams.setJdbcUriString(uri); // Create a Java URI from the resolved URI for extracting the host/port @@ -356,6 +389,26 @@ public static JdbcConnectionParams parseURL(String uri) throws JdbcUriParseExcep } /** + * Remove the deprecatedName param from the fromMap and put the key value in the toMap. + * Also log a deprecation message for the client. + * @param fromMap + * @param toMap + * @param deprecatedName + * @param newName + * @param newUsage + */ + private static void handleParamDeprecation(Map fromMap, Map toMap, + String deprecatedName, String newName, String newUsage) { + if (fromMap.containsKey(deprecatedName)) { + LOG.warn("***** JDBC param deprecation *****"); + LOG.warn("The use of " + deprecatedName + " is deprecated."); + LOG.warn("Please use " + newName +" like so: " + newUsage); + String paramValue = fromMap.remove(deprecatedName); + toMap.put(newName, paramValue); + } + } + + /** * Get the authority string from the supplied uri, which could potentially contain multiple * host:port pairs. * @@ -367,14 +420,30 @@ public static JdbcConnectionParams parseURL(String uri) throws JdbcUriParseExcep private static String getAuthorities(String uri, JdbcConnectionParams connParams) throws JdbcUriParseException { String authorities; - // For a jdbc uri like: jdbc:hive2://host1:port1,host2:port2,host3:port3/ - // Extract the uri host:port list starting after "jdbc:hive2://", till the 1st "/" or EOL + /** + * For a jdbc uri like: + * jdbc:hive2://:,:/dbName;sess_var_list?conf_list#var_list + * Extract the uri host:port list starting after "jdbc:hive2://", + * till the 1st "/" or "?" or "#" whichever comes first & in the given order + * Examples: + * jdbc:hive2://host1:port1,host2:port2,host3:port3/db;k1=v1?k2=v2#k3=v3 + * jdbc:hive2://host1:port1,host2:port2,host3:port3/;k1=v1?k2=v2#k3=v3 + * jdbc:hive2://host1:port1,host2:port2,host3:port3?k2=v2#k3=v3 + * jdbc:hive2://host1:port1,host2:port2,host3:port3#k3=v3 + */ int fromIndex = Utils.URL_PREFIX.length(); - int toIndex = uri.indexOf("/", fromIndex); + int toIndex = -1; + ArrayList toIndexChars = new ArrayList(Arrays.asList("/", "?", "#")); + for (String toIndexChar : toIndexChars) { + toIndex = uri.indexOf(toIndexChar, fromIndex); + if (toIndex > 0) { + break; + } + } if (toIndex < 0) { authorities = uri.substring(fromIndex); } else { - authorities = uri.substring(fromIndex, uri.indexOf("/", fromIndex)); + authorities = uri.substring(fromIndex, toIndex); } return authorities; } diff --git a/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java b/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java index 06795a5..3e4f4e5 100644 --- a/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java +++ b/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java @@ -53,12 +53,16 @@ static String getNextServerUriFromZooKeeper(JdbcConnectionParams connParams) String zooKeeperEnsemble = connParams.getZooKeeperEnsemble(); String zooKeeperNamespace = connParams.getSessionVars().get(JdbcConnectionParams.ZOOKEEPER_NAMESPACE); + if ((zooKeeperNamespace == null) || (zooKeeperNamespace.isEmpty())) { + zooKeeperNamespace = JdbcConnectionParams.ZOOKEEPER_DEFAULT_NAMESPACE; + } List serverHosts; Random randomizer = new Random(); String serverNode; + ZooKeeper zooKeeperClient = null; // Pick a random HiveServer2 host from the ZooKeeper namspace try { - ZooKeeper zooKeeperClient = + zooKeeperClient = new ZooKeeper(zooKeeperEnsemble, JdbcConnectionParams.ZOOKEEPER_SESSION_TIMEOUT, new ZooKeeperHiveClientHelper.DummyWatcher()); // All the HiveServer2 host nodes that are in ZooKeeper currently @@ -80,7 +84,15 @@ static String getNextServerUriFromZooKeeper(JdbcConnectionParams connParams) return serverUri; } catch (Exception e) { throw new ZooKeeperHiveClientException("Unable to read HiveServer2 uri from ZooKeeper", e); + } finally { + // Try to close the client connection with ZooKeeper + if (zooKeeperClient != null) { + try { + zooKeeperClient.close(); + } catch (Exception e) { + // No-op + } + } } } - } diff --git a/metastore/pom.xml b/metastore/pom.xml index 5c7d4eb..948f22c 100644 --- a/metastore/pom.xml +++ b/metastore/pom.xml @@ -19,7 +19,7 @@ org.apache.hive hive - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../pom.xml diff --git a/metastore/scripts/upgrade/derby/hive-schema-0.14.0.derby.sql b/metastore/scripts/upgrade/derby/hive-schema-0.14.0.derby.sql index 90d5e9b..1d0a63b 100644 --- a/metastore/scripts/upgrade/derby/hive-schema-0.14.0.derby.sql +++ b/metastore/scripts/upgrade/derby/hive-schema-0.14.0.derby.sql @@ -318,8 +318,75 @@ ALTER TABLE "APP"."SDS" ADD CONSTRAINT "SQL110318025505550" CHECK (IS_COMPRESSED -- ---------------------------- -- Transaction and Lock Tables +-- These are not part of package jdo, so if you are going to regenerate this file you need to manually add the following section back to the file. -- ---------------------------- -RUN 'hive-txn-schema-0.13.0.derby.sql'; +CREATE TABLE TXNS ( + TXN_ID bigint PRIMARY KEY, + TXN_STATE char(1) NOT NULL, + TXN_STARTED bigint NOT NULL, + TXN_LAST_HEARTBEAT bigint NOT NULL, + TXN_USER varchar(128) NOT NULL, + TXN_HOST varchar(128) NOT NULL +); + +CREATE TABLE TXN_COMPONENTS ( + TC_TXNID bigint REFERENCES TXNS (TXN_ID), + TC_DATABASE varchar(128) NOT NULL, + TC_TABLE varchar(128), + TC_PARTITION varchar(767) +); + +CREATE TABLE COMPLETED_TXN_COMPONENTS ( + CTC_TXNID bigint, + CTC_DATABASE varchar(128) NOT NULL, + CTC_TABLE varchar(128), + CTC_PARTITION varchar(767) +); + +CREATE TABLE NEXT_TXN_ID ( + NTXN_NEXT bigint NOT NULL +); +INSERT INTO NEXT_TXN_ID VALUES(1); + +CREATE TABLE HIVE_LOCKS ( + HL_LOCK_EXT_ID bigint NOT NULL, + HL_LOCK_INT_ID bigint NOT NULL, + HL_TXNID bigint, + HL_DB varchar(128) NOT NULL, + HL_TABLE varchar(128), + HL_PARTITION varchar(767), + HL_LOCK_STATE char(1) NOT NULL, + HL_LOCK_TYPE char(1) NOT NULL, + HL_LAST_HEARTBEAT bigint NOT NULL, + HL_ACQUIRED_AT bigint, + HL_USER varchar(128) NOT NULL, + HL_HOST varchar(128) NOT NULL, + PRIMARY KEY(HL_LOCK_EXT_ID, HL_LOCK_INT_ID) +); + +CREATE INDEX HL_TXNID_INDEX ON HIVE_LOCKS (HL_TXNID); + +CREATE TABLE NEXT_LOCK_ID ( + NL_NEXT bigint NOT NULL +); +INSERT INTO NEXT_LOCK_ID VALUES(1); + +CREATE TABLE COMPACTION_QUEUE ( + CQ_ID bigint PRIMARY KEY, + CQ_DATABASE varchar(128) NOT NULL, + CQ_TABLE varchar(128) NOT NULL, + CQ_PARTITION varchar(767), + CQ_STATE char(1) NOT NULL, + CQ_TYPE char(1) NOT NULL, + CQ_WORKER_ID varchar(128), + CQ_START bigint, + CQ_RUN_AS varchar(128) +); + +CREATE TABLE NEXT_COMPACTION_QUEUE_ID ( + NCQ_NEXT bigint NOT NULL +); +INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1); -- ----------------------------------------------------------------- -- Record schema version. Should be the last step in the init script diff --git a/metastore/scripts/upgrade/derby/hive-schema-0.15.0.derby.sql b/metastore/scripts/upgrade/derby/hive-schema-0.15.0.derby.sql new file mode 100644 index 0000000..a239f32 --- /dev/null +++ b/metastore/scripts/upgrade/derby/hive-schema-0.15.0.derby.sql @@ -0,0 +1,328 @@ +-- Timestamp: 2011-09-22 15:32:02.024 +-- Source database is: /home/carl/Work/repos/hive1/metastore/scripts/upgrade/derby/mdb +-- Connection URL is: jdbc:derby:/home/carl/Work/repos/hive1/metastore/scripts/upgrade/derby/mdb +-- Specified schema is: APP +-- appendLogs: false + +-- ---------------------------------------------- +-- DDL Statements for functions +-- ---------------------------------------------- + +CREATE FUNCTION "APP"."NUCLEUS_ASCII" (C CHAR(1)) RETURNS INTEGER LANGUAGE JAVA PARAMETER STYLE JAVA READS SQL DATA CALLED ON NULL INPUT EXTERNAL NAME 'org.datanucleus.store.rdbms.adapter.DerbySQLFunction.ascii' ; + +CREATE FUNCTION "APP"."NUCLEUS_MATCHES" (TEXT VARCHAR(8000),PATTERN VARCHAR(8000)) RETURNS INTEGER LANGUAGE JAVA PARAMETER STYLE JAVA READS SQL DATA CALLED ON NULL INPUT EXTERNAL NAME 'org.datanucleus.store.rdbms.adapter.DerbySQLFunction.matches' ; + +-- ---------------------------------------------- +-- DDL Statements for tables +-- ---------------------------------------------- + +CREATE TABLE "APP"."DBS" ("DB_ID" BIGINT NOT NULL, "DESC" VARCHAR(4000), "DB_LOCATION_URI" VARCHAR(4000) NOT NULL, "NAME" VARCHAR(128), "OWNER_NAME" VARCHAR(128), "OWNER_TYPE" VARCHAR(10)); + +CREATE TABLE "APP"."TBL_PRIVS" ("TBL_GRANT_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "GRANT_OPTION" SMALLINT NOT NULL, "GRANTOR" VARCHAR(128), "GRANTOR_TYPE" VARCHAR(128), "PRINCIPAL_NAME" VARCHAR(128), "PRINCIPAL_TYPE" VARCHAR(128), "TBL_PRIV" VARCHAR(128), "TBL_ID" BIGINT); + +CREATE TABLE "APP"."DATABASE_PARAMS" ("DB_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(180) NOT NULL, "PARAM_VALUE" VARCHAR(4000)); + +CREATE TABLE "APP"."TBL_COL_PRIVS" ("TBL_COLUMN_GRANT_ID" BIGINT NOT NULL, "COLUMN_NAME" VARCHAR(128), "CREATE_TIME" INTEGER NOT NULL, "GRANT_OPTION" SMALLINT NOT NULL, "GRANTOR" VARCHAR(128), "GRANTOR_TYPE" VARCHAR(128), "PRINCIPAL_NAME" VARCHAR(128), "PRINCIPAL_TYPE" VARCHAR(128), "TBL_COL_PRIV" VARCHAR(128), "TBL_ID" BIGINT); + +CREATE TABLE "APP"."SERDE_PARAMS" ("SERDE_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" VARCHAR(4000)); + +CREATE TABLE "APP"."COLUMNS_V2" ("CD_ID" BIGINT NOT NULL, "COMMENT" VARCHAR(4000), "COLUMN_NAME" VARCHAR(128) NOT NULL, "TYPE_NAME" VARCHAR(4000), "INTEGER_IDX" INTEGER NOT NULL); + +CREATE TABLE "APP"."SORT_COLS" ("SD_ID" BIGINT NOT NULL, "COLUMN_NAME" VARCHAR(128), "ORDER" INTEGER NOT NULL, "INTEGER_IDX" INTEGER NOT NULL); + +CREATE TABLE "APP"."CDS" ("CD_ID" BIGINT NOT NULL); + +CREATE TABLE "APP"."PARTITION_KEY_VALS" ("PART_ID" BIGINT NOT NULL, "PART_KEY_VAL" VARCHAR(256), "INTEGER_IDX" INTEGER NOT NULL); + +CREATE TABLE "APP"."DB_PRIVS" ("DB_GRANT_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "DB_ID" BIGINT, "GRANT_OPTION" SMALLINT NOT NULL, "GRANTOR" VARCHAR(128), "GRANTOR_TYPE" VARCHAR(128), "PRINCIPAL_NAME" VARCHAR(128), "PRINCIPAL_TYPE" VARCHAR(128), "DB_PRIV" VARCHAR(128)); + +CREATE TABLE "APP"."IDXS" ("INDEX_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "DEFERRED_REBUILD" CHAR(1) NOT NULL, "INDEX_HANDLER_CLASS" VARCHAR(4000), "INDEX_NAME" VARCHAR(128), "INDEX_TBL_ID" BIGINT, "LAST_ACCESS_TIME" INTEGER NOT NULL, "ORIG_TBL_ID" BIGINT, "SD_ID" BIGINT); + +CREATE TABLE "APP"."INDEX_PARAMS" ("INDEX_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" VARCHAR(4000)); + +CREATE TABLE "APP"."PARTITIONS" ("PART_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "LAST_ACCESS_TIME" INTEGER NOT NULL, "PART_NAME" VARCHAR(767), "SD_ID" BIGINT, "TBL_ID" BIGINT); + +CREATE TABLE "APP"."SERDES" ("SERDE_ID" BIGINT NOT NULL, "NAME" VARCHAR(128), "SLIB" VARCHAR(4000)); + +CREATE TABLE "APP"."PART_PRIVS" ("PART_GRANT_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "GRANT_OPTION" SMALLINT NOT NULL, "GRANTOR" VARCHAR(128), "GRANTOR_TYPE" VARCHAR(128), "PART_ID" BIGINT, "PRINCIPAL_NAME" VARCHAR(128), "PRINCIPAL_TYPE" VARCHAR(128), "PART_PRIV" VARCHAR(128)); + +CREATE TABLE "APP"."ROLE_MAP" ("ROLE_GRANT_ID" BIGINT NOT NULL, "ADD_TIME" INTEGER NOT NULL, "GRANT_OPTION" SMALLINT NOT NULL, "GRANTOR" VARCHAR(128), "GRANTOR_TYPE" VARCHAR(128), "PRINCIPAL_NAME" VARCHAR(128), "PRINCIPAL_TYPE" VARCHAR(128), "ROLE_ID" BIGINT); + +CREATE TABLE "APP"."TYPES" ("TYPES_ID" BIGINT NOT NULL, "TYPE_NAME" VARCHAR(128), "TYPE1" VARCHAR(767), "TYPE2" VARCHAR(767)); + +CREATE TABLE "APP"."GLOBAL_PRIVS" ("USER_GRANT_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "GRANT_OPTION" SMALLINT NOT NULL, "GRANTOR" VARCHAR(128), "GRANTOR_TYPE" VARCHAR(128), "PRINCIPAL_NAME" VARCHAR(128), "PRINCIPAL_TYPE" VARCHAR(128), "USER_PRIV" VARCHAR(128)); + +CREATE TABLE "APP"."PARTITION_PARAMS" ("PART_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" VARCHAR(4000)); + +CREATE TABLE "APP"."PARTITION_EVENTS" ("PART_NAME_ID" BIGINT NOT NULL, "DB_NAME" VARCHAR(128), "EVENT_TIME" BIGINT NOT NULL, "EVENT_TYPE" INTEGER NOT NULL, "PARTITION_NAME" VARCHAR(767), "TBL_NAME" VARCHAR(128)); + +CREATE TABLE "APP"."COLUMNS" ("SD_ID" BIGINT NOT NULL, "COMMENT" VARCHAR(256), "COLUMN_NAME" VARCHAR(128) NOT NULL, "TYPE_NAME" VARCHAR(4000) NOT NULL, "INTEGER_IDX" INTEGER NOT NULL); + +CREATE TABLE "APP"."ROLES" ("ROLE_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "OWNER_NAME" VARCHAR(128), "ROLE_NAME" VARCHAR(128)); + +CREATE TABLE "APP"."TBLS" ("TBL_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "DB_ID" BIGINT, "LAST_ACCESS_TIME" INTEGER NOT NULL, "OWNER" VARCHAR(767), "RETENTION" INTEGER NOT NULL, "SD_ID" BIGINT, "TBL_NAME" VARCHAR(128), "TBL_TYPE" VARCHAR(128), "VIEW_EXPANDED_TEXT" LONG VARCHAR, "VIEW_ORIGINAL_TEXT" LONG VARCHAR); + +CREATE TABLE "APP"."PARTITION_KEYS" ("TBL_ID" BIGINT NOT NULL, "PKEY_COMMENT" VARCHAR(4000), "PKEY_NAME" VARCHAR(128) NOT NULL, "PKEY_TYPE" VARCHAR(767) NOT NULL, "INTEGER_IDX" INTEGER NOT NULL); + +CREATE TABLE "APP"."PART_COL_PRIVS" ("PART_COLUMN_GRANT_ID" BIGINT NOT NULL, "COLUMN_NAME" VARCHAR(128), "CREATE_TIME" INTEGER NOT NULL, "GRANT_OPTION" SMALLINT NOT NULL, "GRANTOR" VARCHAR(128), "GRANTOR_TYPE" VARCHAR(128), "PART_ID" BIGINT, "PRINCIPAL_NAME" VARCHAR(128), "PRINCIPAL_TYPE" VARCHAR(128), "PART_COL_PRIV" VARCHAR(128)); + +CREATE TABLE "APP"."SDS" ("SD_ID" BIGINT NOT NULL, "INPUT_FORMAT" VARCHAR(4000), "IS_COMPRESSED" CHAR(1) NOT NULL, "LOCATION" VARCHAR(4000), "NUM_BUCKETS" INTEGER NOT NULL, "OUTPUT_FORMAT" VARCHAR(4000), "SERDE_ID" BIGINT, "CD_ID" BIGINT, "IS_STOREDASSUBDIRECTORIES" CHAR(1) NOT NULL); + +CREATE TABLE "APP"."SEQUENCE_TABLE" ("SEQUENCE_NAME" VARCHAR(256) NOT NULL, "NEXT_VAL" BIGINT NOT NULL); + +CREATE TABLE "APP"."TABLE_PARAMS" ("TBL_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" VARCHAR(4000)); + +CREATE TABLE "APP"."BUCKETING_COLS" ("SD_ID" BIGINT NOT NULL, "BUCKET_COL_NAME" VARCHAR(256), "INTEGER_IDX" INTEGER NOT NULL); + +CREATE TABLE "APP"."TYPE_FIELDS" ("TYPE_NAME" BIGINT NOT NULL, "COMMENT" VARCHAR(256), "FIELD_NAME" VARCHAR(128) NOT NULL, "FIELD_TYPE" VARCHAR(767) NOT NULL, "INTEGER_IDX" INTEGER NOT NULL); + +CREATE TABLE "APP"."NUCLEUS_TABLES" ("CLASS_NAME" VARCHAR(128) NOT NULL, "TABLE_NAME" VARCHAR(128) NOT NULL, "TYPE" VARCHAR(4) NOT NULL, "OWNER" VARCHAR(2) NOT NULL, "VERSION" VARCHAR(20) NOT NULL, "INTERFACE_NAME" VARCHAR(256) DEFAULT NULL); + +CREATE TABLE "APP"."SD_PARAMS" ("SD_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" VARCHAR(4000)); + +CREATE TABLE "APP"."SKEWED_STRING_LIST" ("STRING_LIST_ID" BIGINT NOT NULL); + +CREATE TABLE "APP"."SKEWED_STRING_LIST_VALUES" ("STRING_LIST_ID" BIGINT NOT NULL, "STRING_LIST_VALUE" VARCHAR(256), "INTEGER_IDX" INTEGER NOT NULL); + +CREATE TABLE "APP"."SKEWED_COL_NAMES" ("SD_ID" BIGINT NOT NULL, "SKEWED_COL_NAME" VARCHAR(256), "INTEGER_IDX" INTEGER NOT NULL); + +CREATE TABLE "APP"."SKEWED_COL_VALUE_LOC_MAP" ("SD_ID" BIGINT NOT NULL, "STRING_LIST_ID_KID" BIGINT NOT NULL, "LOCATION" VARCHAR(4000)); + +CREATE TABLE "APP"."SKEWED_VALUES" ("SD_ID_OID" BIGINT NOT NULL, "STRING_LIST_ID_EID" BIGINT NOT NULL, "INTEGER_IDX" INTEGER NOT NULL); + +CREATE TABLE "APP"."MASTER_KEYS" ("KEY_ID" INTEGER NOT NULL generated always as identity (start with 1), "MASTER_KEY" VARCHAR(767)); + +CREATE TABLE "APP"."DELEGATION_TOKENS" ( "TOKEN_IDENT" VARCHAR(767) NOT NULL, "TOKEN" VARCHAR(767)); + +CREATE TABLE "APP"."TAB_COL_STATS"("DB_NAME" VARCHAR(128) NOT NULL,"TABLE_NAME" VARCHAR(128) NOT NULL, "COLUMN_NAME" VARCHAR(128) NOT NULL, "COLUMN_TYPE" VARCHAR(128) NOT NULL, "LONG_LOW_VALUE" BIGINT, "LONG_HIGH_VALUE" BIGINT, "DOUBLE_LOW_VALUE" DOUBLE, "DOUBLE_HIGH_VALUE" DOUBLE, "BIG_DECIMAL_LOW_VALUE" VARCHAR(4000), "BIG_DECIMAL_HIGH_VALUE" VARCHAR(4000),"NUM_DISTINCTS" BIGINT, "NUM_NULLS" BIGINT NOT NULL, "AVG_COL_LEN" DOUBLE, "MAX_COL_LEN" BIGINT, "NUM_TRUES" BIGINT, "NUM_FALSES" BIGINT, "LAST_ANALYZED" BIGINT, "CS_ID" BIGINT NOT NULL, "TBL_ID" BIGINT NOT NULL); + +CREATE TABLE "APP"."PART_COL_STATS"("DB_NAME" VARCHAR(128) NOT NULL,"TABLE_NAME" VARCHAR(128) NOT NULL, "PARTITION_NAME" VARCHAR(767) NOT NULL, "COLUMN_NAME" VARCHAR(128) NOT NULL, "COLUMN_TYPE" VARCHAR(128) NOT NULL, "LONG_LOW_VALUE" BIGINT, "LONG_HIGH_VALUE" BIGINT, "DOUBLE_LOW_VALUE" DOUBLE, "DOUBLE_HIGH_VALUE" DOUBLE, "BIG_DECIMAL_LOW_VALUE" VARCHAR(4000), "BIG_DECIMAL_HIGH_VALUE" VARCHAR(4000),"NUM_DISTINCTS" BIGINT, "NUM_NULLS" BIGINT NOT NULL, "AVG_COL_LEN" DOUBLE, "MAX_COL_LEN" BIGINT, "NUM_TRUES" BIGINT, "NUM_FALSES" BIGINT, "LAST_ANALYZED" BIGINT, "CS_ID" BIGINT NOT NULL, "PART_ID" BIGINT NOT NULL); + +CREATE TABLE "APP"."VERSION" ("VER_ID" BIGINT NOT NULL, "SCHEMA_VERSION" VARCHAR(127) NOT NULL, "VERSION_COMMENT" VARCHAR(255)); + +CREATE TABLE "APP"."FUNCS" ("FUNC_ID" BIGINT NOT NULL, "CLASS_NAME" VARCHAR(4000), "CREATE_TIME" INTEGER NOT NULL, "DB_ID" BIGINT, "FUNC_NAME" VARCHAR(128), "FUNC_TYPE" INTEGER NOT NULL, "OWNER_NAME" VARCHAR(128), "OWNER_TYPE" VARCHAR(10)); + +CREATE TABLE "APP"."FUNC_RU" ("FUNC_ID" BIGINT NOT NULL, "RESOURCE_TYPE" INTEGER NOT NULL, "RESOURCE_URI" VARCHAR(4000), "INTEGER_IDX" INTEGER NOT NULL); + +-- ---------------------------------------------- +-- DDL Statements for indexes +-- ---------------------------------------------- + +CREATE UNIQUE INDEX "APP"."UNIQUEINDEX" ON "APP"."IDXS" ("INDEX_NAME", "ORIG_TBL_ID"); + +CREATE INDEX "APP"."TABLECOLUMNPRIVILEGEINDEX" ON "APP"."TBL_COL_PRIVS" ("TBL_ID", "COLUMN_NAME", "PRINCIPAL_NAME", "PRINCIPAL_TYPE", "TBL_COL_PRIV", "GRANTOR", "GRANTOR_TYPE"); + +CREATE UNIQUE INDEX "APP"."DBPRIVILEGEINDEX" ON "APP"."DB_PRIVS" ("DB_ID", "PRINCIPAL_NAME", "PRINCIPAL_TYPE", "DB_PRIV", "GRANTOR", "GRANTOR_TYPE"); + +CREATE INDEX "APP"."PCS_STATS_IDX" ON "APP"."PART_COL_STATS" ("DB_NAME","TABLE_NAME","COLUMN_NAME","PARTITION_NAME"); + +CREATE INDEX "APP"."PARTPRIVILEGEINDEX" ON "APP"."PART_PRIVS" ("PART_ID", "PRINCIPAL_NAME", "PRINCIPAL_TYPE", "PART_PRIV", "GRANTOR", "GRANTOR_TYPE"); + +CREATE UNIQUE INDEX "APP"."ROLEENTITYINDEX" ON "APP"."ROLES" ("ROLE_NAME"); + +CREATE INDEX "APP"."TABLEPRIVILEGEINDEX" ON "APP"."TBL_PRIVS" ("TBL_ID", "PRINCIPAL_NAME", "PRINCIPAL_TYPE", "TBL_PRIV", "GRANTOR", "GRANTOR_TYPE"); + +CREATE UNIQUE INDEX "APP"."UNIQUETABLE" ON "APP"."TBLS" ("TBL_NAME", "DB_ID"); + +CREATE UNIQUE INDEX "APP"."UNIQUE_DATABASE" ON "APP"."DBS" ("NAME"); + +CREATE UNIQUE INDEX "APP"."USERROLEMAPINDEX" ON "APP"."ROLE_MAP" ("PRINCIPAL_NAME", "ROLE_ID", "GRANTOR", "GRANTOR_TYPE"); + +CREATE UNIQUE INDEX "APP"."GLOBALPRIVILEGEINDEX" ON "APP"."GLOBAL_PRIVS" ("PRINCIPAL_NAME", "PRINCIPAL_TYPE", "USER_PRIV", "GRANTOR", "GRANTOR_TYPE"); + +CREATE UNIQUE INDEX "APP"."UNIQUE_TYPE" ON "APP"."TYPES" ("TYPE_NAME"); + +CREATE INDEX "APP"."PARTITIONCOLUMNPRIVILEGEINDEX" ON "APP"."PART_COL_PRIVS" ("PART_ID", "COLUMN_NAME", "PRINCIPAL_NAME", "PRINCIPAL_TYPE", "PART_COL_PRIV", "GRANTOR", "GRANTOR_TYPE"); + +CREATE UNIQUE INDEX "APP"."UNIQUEPARTITION" ON "APP"."PARTITIONS" ("PART_NAME", "TBL_ID"); + +CREATE UNIQUE INDEX "APP"."UNIQUEFUNCTION" ON "APP"."FUNCS" ("FUNC_NAME", "DB_ID"); + +CREATE INDEX "APP"."FUNCS_N49" ON "APP"."FUNCS" ("DB_ID"); + +CREATE INDEX "APP"."FUNC_RU_N49" ON "APP"."FUNC_RU" ("FUNC_ID"); + +-- ---------------------------------------------- +-- DDL Statements for keys +-- ---------------------------------------------- + +-- primary/unique +ALTER TABLE "APP"."IDXS" ADD CONSTRAINT "IDXS_PK" PRIMARY KEY ("INDEX_ID"); + +ALTER TABLE "APP"."TBL_COL_PRIVS" ADD CONSTRAINT "TBL_COL_PRIVS_PK" PRIMARY KEY ("TBL_COLUMN_GRANT_ID"); + +ALTER TABLE "APP"."CDS" ADD CONSTRAINT "SQL110922153006460" PRIMARY KEY ("CD_ID"); + +ALTER TABLE "APP"."DB_PRIVS" ADD CONSTRAINT "DB_PRIVS_PK" PRIMARY KEY ("DB_GRANT_ID"); + +ALTER TABLE "APP"."INDEX_PARAMS" ADD CONSTRAINT "INDEX_PARAMS_PK" PRIMARY KEY ("INDEX_ID", "PARAM_KEY"); + +ALTER TABLE "APP"."PARTITION_KEYS" ADD CONSTRAINT "PARTITION_KEY_PK" PRIMARY KEY ("TBL_ID", "PKEY_NAME"); + +ALTER TABLE "APP"."SEQUENCE_TABLE" ADD CONSTRAINT "SEQUENCE_TABLE_PK" PRIMARY KEY ("SEQUENCE_NAME"); + +ALTER TABLE "APP"."PART_PRIVS" ADD CONSTRAINT "PART_PRIVS_PK" PRIMARY KEY ("PART_GRANT_ID"); + +ALTER TABLE "APP"."SDS" ADD CONSTRAINT "SDS_PK" PRIMARY KEY ("SD_ID"); + +ALTER TABLE "APP"."SERDES" ADD CONSTRAINT "SERDES_PK" PRIMARY KEY ("SERDE_ID"); + +ALTER TABLE "APP"."COLUMNS" ADD CONSTRAINT "COLUMNS_PK" PRIMARY KEY ("SD_ID", "COLUMN_NAME"); + +ALTER TABLE "APP"."PARTITION_EVENTS" ADD CONSTRAINT "PARTITION_EVENTS_PK" PRIMARY KEY ("PART_NAME_ID"); + +ALTER TABLE "APP"."TYPE_FIELDS" ADD CONSTRAINT "TYPE_FIELDS_PK" PRIMARY KEY ("TYPE_NAME", "FIELD_NAME"); + +ALTER TABLE "APP"."ROLES" ADD CONSTRAINT "ROLES_PK" PRIMARY KEY ("ROLE_ID"); + +ALTER TABLE "APP"."TBL_PRIVS" ADD CONSTRAINT "TBL_PRIVS_PK" PRIMARY KEY ("TBL_GRANT_ID"); + +ALTER TABLE "APP"."SERDE_PARAMS" ADD CONSTRAINT "SERDE_PARAMS_PK" PRIMARY KEY ("SERDE_ID", "PARAM_KEY"); + +ALTER TABLE "APP"."NUCLEUS_TABLES" ADD CONSTRAINT "NUCLEUS_TABLES_PK" PRIMARY KEY ("CLASS_NAME"); + +ALTER TABLE "APP"."TBLS" ADD CONSTRAINT "TBLS_PK" PRIMARY KEY ("TBL_ID"); + +ALTER TABLE "APP"."SD_PARAMS" ADD CONSTRAINT "SD_PARAMS_PK" PRIMARY KEY ("SD_ID", "PARAM_KEY"); + +ALTER TABLE "APP"."DATABASE_PARAMS" ADD CONSTRAINT "DATABASE_PARAMS_PK" PRIMARY KEY ("DB_ID", "PARAM_KEY"); + +ALTER TABLE "APP"."DBS" ADD CONSTRAINT "DBS_PK" PRIMARY KEY ("DB_ID"); + +ALTER TABLE "APP"."ROLE_MAP" ADD CONSTRAINT "ROLE_MAP_PK" PRIMARY KEY ("ROLE_GRANT_ID"); + +ALTER TABLE "APP"."GLOBAL_PRIVS" ADD CONSTRAINT "GLOBAL_PRIVS_PK" PRIMARY KEY ("USER_GRANT_ID"); + +ALTER TABLE "APP"."BUCKETING_COLS" ADD CONSTRAINT "BUCKETING_COLS_PK" PRIMARY KEY ("SD_ID", "INTEGER_IDX"); + +ALTER TABLE "APP"."SORT_COLS" ADD CONSTRAINT "SORT_COLS_PK" PRIMARY KEY ("SD_ID", "INTEGER_IDX"); + +ALTER TABLE "APP"."PARTITION_KEY_VALS" ADD CONSTRAINT "PARTITION_KEY_VALS_PK" PRIMARY KEY ("PART_ID", "INTEGER_IDX"); + +ALTER TABLE "APP"."TYPES" ADD CONSTRAINT "TYPES_PK" PRIMARY KEY ("TYPES_ID"); + +ALTER TABLE "APP"."COLUMNS_V2" ADD CONSTRAINT "SQL110922153006740" PRIMARY KEY ("CD_ID", "COLUMN_NAME"); + +ALTER TABLE "APP"."PART_COL_PRIVS" ADD CONSTRAINT "PART_COL_PRIVS_PK" PRIMARY KEY ("PART_COLUMN_GRANT_ID"); + +ALTER TABLE "APP"."PARTITION_PARAMS" ADD CONSTRAINT "PARTITION_PARAMS_PK" PRIMARY KEY ("PART_ID", "PARAM_KEY"); + +ALTER TABLE "APP"."PARTITIONS" ADD CONSTRAINT "PARTITIONS_PK" PRIMARY KEY ("PART_ID"); + +ALTER TABLE "APP"."TABLE_PARAMS" ADD CONSTRAINT "TABLE_PARAMS_PK" PRIMARY KEY ("TBL_ID", "PARAM_KEY"); + +ALTER TABLE "APP"."SKEWED_STRING_LIST" ADD CONSTRAINT "SKEWED_STRING_LIST_PK" PRIMARY KEY ("STRING_LIST_ID"); + +ALTER TABLE "APP"."SKEWED_STRING_LIST_VALUES" ADD CONSTRAINT "SKEWED_STRING_LIST_VALUES_PK" PRIMARY KEY ("STRING_LIST_ID", "INTEGER_IDX"); + +ALTER TABLE "APP"."SKEWED_COL_NAMES" ADD CONSTRAINT "SKEWED_COL_NAMES_PK" PRIMARY KEY ("SD_ID", "INTEGER_IDX"); + +ALTER TABLE "APP"."SKEWED_COL_VALUE_LOC_MAP" ADD CONSTRAINT "SKEWED_COL_VALUE_LOC_MAP_PK" PRIMARY KEY ("SD_ID", "STRING_LIST_ID_KID"); + +ALTER TABLE "APP"."SKEWED_VALUES" ADD CONSTRAINT "SKEWED_VALUES_PK" PRIMARY KEY ("SD_ID_OID", "INTEGER_IDX"); + +ALTER TABLE "APP"."TAB_COL_STATS" ADD CONSTRAINT "TAB_COL_STATS_PK" PRIMARY KEY ("CS_ID"); + +ALTER TABLE "APP"."PART_COL_STATS" ADD CONSTRAINT "PART_COL_STATS_PK" PRIMARY KEY ("CS_ID"); + +ALTER TABLE "APP"."FUNCS" ADD CONSTRAINT "FUNCS_PK" PRIMARY KEY ("FUNC_ID"); + +ALTER TABLE "APP"."FUNC_RU" ADD CONSTRAINT "FUNC_RU_PK" PRIMARY KEY ("FUNC_ID", "INTEGER_IDX"); + +-- foreign +ALTER TABLE "APP"."IDXS" ADD CONSTRAINT "IDXS_FK1" FOREIGN KEY ("ORIG_TBL_ID") REFERENCES "APP"."TBLS" ("TBL_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."IDXS" ADD CONSTRAINT "IDXS_FK2" FOREIGN KEY ("SD_ID") REFERENCES "APP"."SDS" ("SD_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."IDXS" ADD CONSTRAINT "IDXS_FK3" FOREIGN KEY ("INDEX_TBL_ID") REFERENCES "APP"."TBLS" ("TBL_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."TBL_COL_PRIVS" ADD CONSTRAINT "TBL_COL_PRIVS_FK1" FOREIGN KEY ("TBL_ID") REFERENCES "APP"."TBLS" ("TBL_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."DB_PRIVS" ADD CONSTRAINT "DB_PRIVS_FK1" FOREIGN KEY ("DB_ID") REFERENCES "APP"."DBS" ("DB_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."INDEX_PARAMS" ADD CONSTRAINT "INDEX_PARAMS_FK1" FOREIGN KEY ("INDEX_ID") REFERENCES "APP"."IDXS" ("INDEX_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."PARTITION_KEYS" ADD CONSTRAINT "PARTITION_KEYS_FK1" FOREIGN KEY ("TBL_ID") REFERENCES "APP"."TBLS" ("TBL_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."PART_PRIVS" ADD CONSTRAINT "PART_PRIVS_FK1" FOREIGN KEY ("PART_ID") REFERENCES "APP"."PARTITIONS" ("PART_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."SDS" ADD CONSTRAINT "SDS_FK1" FOREIGN KEY ("SERDE_ID") REFERENCES "APP"."SERDES" ("SERDE_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."SDS" ADD CONSTRAINT "SDS_FK2" FOREIGN KEY ("CD_ID") REFERENCES "APP"."CDS" ("CD_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."COLUMNS" ADD CONSTRAINT "COLUMNS_FK1" FOREIGN KEY ("SD_ID") REFERENCES "APP"."SDS" ("SD_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."TYPE_FIELDS" ADD CONSTRAINT "TYPE_FIELDS_FK1" FOREIGN KEY ("TYPE_NAME") REFERENCES "APP"."TYPES" ("TYPES_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."TBL_PRIVS" ADD CONSTRAINT "TBL_PRIVS_FK1" FOREIGN KEY ("TBL_ID") REFERENCES "APP"."TBLS" ("TBL_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."SERDE_PARAMS" ADD CONSTRAINT "SERDE_PARAMS_FK1" FOREIGN KEY ("SERDE_ID") REFERENCES "APP"."SERDES" ("SERDE_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."TBLS" ADD CONSTRAINT "TBLS_FK2" FOREIGN KEY ("SD_ID") REFERENCES "APP"."SDS" ("SD_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."TBLS" ADD CONSTRAINT "TBLS_FK1" FOREIGN KEY ("DB_ID") REFERENCES "APP"."DBS" ("DB_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."SD_PARAMS" ADD CONSTRAINT "SD_PARAMS_FK1" FOREIGN KEY ("SD_ID") REFERENCES "APP"."SDS" ("SD_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."DATABASE_PARAMS" ADD CONSTRAINT "DATABASE_PARAMS_FK1" FOREIGN KEY ("DB_ID") REFERENCES "APP"."DBS" ("DB_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."ROLE_MAP" ADD CONSTRAINT "ROLE_MAP_FK1" FOREIGN KEY ("ROLE_ID") REFERENCES "APP"."ROLES" ("ROLE_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."BUCKETING_COLS" ADD CONSTRAINT "BUCKETING_COLS_FK1" FOREIGN KEY ("SD_ID") REFERENCES "APP"."SDS" ("SD_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."SORT_COLS" ADD CONSTRAINT "SORT_COLS_FK1" FOREIGN KEY ("SD_ID") REFERENCES "APP"."SDS" ("SD_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."PARTITION_KEY_VALS" ADD CONSTRAINT "PARTITION_KEY_VALS_FK1" FOREIGN KEY ("PART_ID") REFERENCES "APP"."PARTITIONS" ("PART_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."COLUMNS_V2" ADD CONSTRAINT "COLUMNS_V2_FK1" FOREIGN KEY ("CD_ID") REFERENCES "APP"."CDS" ("CD_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."PART_COL_PRIVS" ADD CONSTRAINT "PART_COL_PRIVS_FK1" FOREIGN KEY ("PART_ID") REFERENCES "APP"."PARTITIONS" ("PART_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."PARTITION_PARAMS" ADD CONSTRAINT "PARTITION_PARAMS_FK1" FOREIGN KEY ("PART_ID") REFERENCES "APP"."PARTITIONS" ("PART_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."PARTITIONS" ADD CONSTRAINT "PARTITIONS_FK1" FOREIGN KEY ("TBL_ID") REFERENCES "APP"."TBLS" ("TBL_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."PARTITIONS" ADD CONSTRAINT "PARTITIONS_FK2" FOREIGN KEY ("SD_ID") REFERENCES "APP"."SDS" ("SD_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."TABLE_PARAMS" ADD CONSTRAINT "TABLE_PARAMS_FK1" FOREIGN KEY ("TBL_ID") REFERENCES "APP"."TBLS" ("TBL_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."SKEWED_STRING_LIST_VALUES" ADD CONSTRAINT "SKEWED_STRING_LIST_VALUES_FK1" FOREIGN KEY ("STRING_LIST_ID") REFERENCES "APP"."SKEWED_STRING_LIST" ("STRING_LIST_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."SKEWED_COL_NAMES" ADD CONSTRAINT "SKEWED_COL_NAMES_FK1" FOREIGN KEY ("SD_ID") REFERENCES "APP"."SDS" ("SD_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."SKEWED_COL_VALUE_LOC_MAP" ADD CONSTRAINT "SKEWED_COL_VALUE_LOC_MAP_FK1" FOREIGN KEY ("SD_ID") REFERENCES "APP"."SDS" ("SD_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."SKEWED_COL_VALUE_LOC_MAP" ADD CONSTRAINT "SKEWED_COL_VALUE_LOC_MAP_FK2" FOREIGN KEY ("STRING_LIST_ID_KID") REFERENCES "APP"."SKEWED_STRING_LIST" ("STRING_LIST_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."SKEWED_VALUES" ADD CONSTRAINT "SKEWED_VALUES_FK1" FOREIGN KEY ("SD_ID_OID") REFERENCES "APP"."SDS" ("SD_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."SKEWED_VALUES" ADD CONSTRAINT "SKEWED_VALUES_FK2" FOREIGN KEY ("STRING_LIST_ID_EID") REFERENCES "APP"."SKEWED_STRING_LIST" ("STRING_LIST_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."TAB_COL_STATS" ADD CONSTRAINT "TAB_COL_STATS_FK" FOREIGN KEY ("TBL_ID") REFERENCES TBLS("TBL_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."PART_COL_STATS" ADD CONSTRAINT "PART_COL_STATS_FK" FOREIGN KEY ("PART_ID") REFERENCES PARTITIONS("PART_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."VERSION" ADD CONSTRAINT "VERSION_PK" PRIMARY KEY ("VER_ID"); + +ALTER TABLE "APP"."FUNCS" ADD CONSTRAINT "FUNCS_FK1" FOREIGN KEY ("DB_ID") REFERENCES "APP"."DBS" ("DB_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +ALTER TABLE "APP"."FUNC_RU" ADD CONSTRAINT "FUNC_RU_FK1" FOREIGN KEY ("FUNC_ID") REFERENCES "APP"."FUNCS" ("FUNC_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +-- ---------------------------------------------- +-- DDL Statements for checks +-- ---------------------------------------------- + +ALTER TABLE "APP"."IDXS" ADD CONSTRAINT "SQL110318025504980" CHECK (DEFERRED_REBUILD IN ('Y','N')); + +ALTER TABLE "APP"."SDS" ADD CONSTRAINT "SQL110318025505550" CHECK (IS_COMPRESSED IN ('Y','N')); + +-- ---------------------------- +-- Transaction and Lock Tables +-- ---------------------------- +RUN 'hive-txn-schema-0.13.0.derby.sql'; + +-- ----------------------------------------------------------------- +-- Record schema version. Should be the last step in the init script +-- ----------------------------------------------------------------- +INSERT INTO "APP"."VERSION" (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES (1, '0.15.0', 'Hive release version 0.15.0'); + diff --git a/metastore/scripts/upgrade/derby/hive-txn-schema-0.14.0.derby.sql b/metastore/scripts/upgrade/derby/hive-txn-schema-0.14.0.derby.sql new file mode 100644 index 0000000..d6c01b6 --- /dev/null +++ b/metastore/scripts/upgrade/derby/hive-txn-schema-0.14.0.derby.sql @@ -0,0 +1,88 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the License); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an AS IS BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- +-- Tables for transaction management +-- +CREATE TABLE TXNS ( + TXN_ID bigint PRIMARY KEY, + TXN_STATE char(1) NOT NULL, + TXN_STARTED bigint NOT NULL, + TXN_LAST_HEARTBEAT bigint NOT NULL, + TXN_USER varchar(128) NOT NULL, + TXN_HOST varchar(128) NOT NULL +); + +CREATE TABLE TXN_COMPONENTS ( + TC_TXNID bigint REFERENCES TXNS (TXN_ID), + TC_DATABASE varchar(128) NOT NULL, + TC_TABLE varchar(128), + TC_PARTITION varchar(767) +); + +CREATE TABLE COMPLETED_TXN_COMPONENTS ( + CTC_TXNID bigint, + CTC_DATABASE varchar(128) NOT NULL, + CTC_TABLE varchar(128), + CTC_PARTITION varchar(767) +); + +CREATE TABLE NEXT_TXN_ID ( + NTXN_NEXT bigint NOT NULL +); +INSERT INTO NEXT_TXN_ID VALUES(1); + +CREATE TABLE HIVE_LOCKS ( + HL_LOCK_EXT_ID bigint NOT NULL, + HL_LOCK_INT_ID bigint NOT NULL, + HL_TXNID bigint, + HL_DB varchar(128) NOT NULL, + HL_TABLE varchar(128), + HL_PARTITION varchar(767), + HL_LOCK_STATE char(1) NOT NULL, + HL_LOCK_TYPE char(1) NOT NULL, + HL_LAST_HEARTBEAT bigint NOT NULL, + HL_ACQUIRED_AT bigint, + HL_USER varchar(128) NOT NULL, + HL_HOST varchar(128) NOT NULL, + PRIMARY KEY(HL_LOCK_EXT_ID, HL_LOCK_INT_ID) +); + +CREATE INDEX HL_TXNID_INDEX ON HIVE_LOCKS (HL_TXNID); + +CREATE TABLE NEXT_LOCK_ID ( + NL_NEXT bigint NOT NULL +); +INSERT INTO NEXT_LOCK_ID VALUES(1); + +CREATE TABLE COMPACTION_QUEUE ( + CQ_ID bigint PRIMARY KEY, + CQ_DATABASE varchar(128) NOT NULL, + CQ_TABLE varchar(128) NOT NULL, + CQ_PARTITION varchar(767), + CQ_STATE char(1) NOT NULL, + CQ_TYPE char(1) NOT NULL, + CQ_WORKER_ID varchar(128), + CQ_START bigint, + CQ_RUN_AS varchar(128) +); + +CREATE TABLE NEXT_COMPACTION_QUEUE_ID ( + NCQ_NEXT bigint NOT NULL +); +INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1); + + + diff --git a/metastore/scripts/upgrade/derby/upgrade-0.14.0-to-0.15.0.derby.sql b/metastore/scripts/upgrade/derby/upgrade-0.14.0-to-0.15.0.derby.sql new file mode 100644 index 0000000..cce9133 --- /dev/null +++ b/metastore/scripts/upgrade/derby/upgrade-0.14.0-to-0.15.0.derby.sql @@ -0,0 +1,3 @@ +-- Upgrade MetaStore schema from 0.14.0 to 0.15.0 + +UPDATE "APP".VERSION SET SCHEMA_VERSION='0.15.0', VERSION_COMMENT='Hive release version 0.15.0' where VER_ID=1; diff --git a/metastore/scripts/upgrade/derby/upgrade.order.derby b/metastore/scripts/upgrade/derby/upgrade.order.derby index 682618a..509e52a 100644 --- a/metastore/scripts/upgrade/derby/upgrade.order.derby +++ b/metastore/scripts/upgrade/derby/upgrade.order.derby @@ -7,3 +7,4 @@ 0.11.0-to-0.12.0 0.12.0-to-0.13.0 0.13.0-to-0.14.0 +0.14.0-to-0.15.0 diff --git a/metastore/scripts/upgrade/mssql/hive-schema-0.15.0.mssql.sql b/metastore/scripts/upgrade/mssql/hive-schema-0.15.0.mssql.sql new file mode 100644 index 0000000..16a9564 --- /dev/null +++ b/metastore/scripts/upgrade/mssql/hive-schema-0.15.0.mssql.sql @@ -0,0 +1,926 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +------------------------------------------------------------------ +-- DataNucleus SchemaTool (ran at 08/04/2014 15:10:15) +------------------------------------------------------------------ +-- Complete schema required for the following classes:- +-- org.apache.hadoop.hive.metastore.model.MColumnDescriptor +-- org.apache.hadoop.hive.metastore.model.MDBPrivilege +-- org.apache.hadoop.hive.metastore.model.MDatabase +-- org.apache.hadoop.hive.metastore.model.MDelegationToken +-- org.apache.hadoop.hive.metastore.model.MFieldSchema +-- org.apache.hadoop.hive.metastore.model.MFunction +-- org.apache.hadoop.hive.metastore.model.MGlobalPrivilege +-- org.apache.hadoop.hive.metastore.model.MIndex +-- org.apache.hadoop.hive.metastore.model.MMasterKey +-- org.apache.hadoop.hive.metastore.model.MOrder +-- org.apache.hadoop.hive.metastore.model.MPartition +-- org.apache.hadoop.hive.metastore.model.MPartitionColumnPrivilege +-- org.apache.hadoop.hive.metastore.model.MPartitionColumnStatistics +-- org.apache.hadoop.hive.metastore.model.MPartitionEvent +-- org.apache.hadoop.hive.metastore.model.MPartitionPrivilege +-- org.apache.hadoop.hive.metastore.model.MResourceUri +-- org.apache.hadoop.hive.metastore.model.MRole +-- org.apache.hadoop.hive.metastore.model.MRoleMap +-- org.apache.hadoop.hive.metastore.model.MSerDeInfo +-- org.apache.hadoop.hive.metastore.model.MStorageDescriptor +-- org.apache.hadoop.hive.metastore.model.MStringList +-- org.apache.hadoop.hive.metastore.model.MTable +-- org.apache.hadoop.hive.metastore.model.MTableColumnPrivilege +-- org.apache.hadoop.hive.metastore.model.MTableColumnStatistics +-- org.apache.hadoop.hive.metastore.model.MTablePrivilege +-- org.apache.hadoop.hive.metastore.model.MType +-- org.apache.hadoop.hive.metastore.model.MVersionTable +-- +-- Table MASTER_KEYS for classes [org.apache.hadoop.hive.metastore.model.MMasterKey] +CREATE TABLE MASTER_KEYS +( + KEY_ID int NOT NULL, + MASTER_KEY varchar(767) NULL +); + +ALTER TABLE MASTER_KEYS ADD CONSTRAINT MASTER_KEYS_PK PRIMARY KEY (KEY_ID); + +-- Table IDXS for classes [org.apache.hadoop.hive.metastore.model.MIndex] +CREATE TABLE IDXS +( + INDEX_ID bigint NOT NULL, + CREATE_TIME int NOT NULL, + DEFERRED_REBUILD bit NOT NULL, + INDEX_HANDLER_CLASS varchar(4000) NULL, + INDEX_NAME varchar(128) NULL, + INDEX_TBL_ID bigint NULL, + LAST_ACCESS_TIME int NOT NULL, + ORIG_TBL_ID bigint NULL, + SD_ID bigint NULL +); + +ALTER TABLE IDXS ADD CONSTRAINT IDXS_PK PRIMARY KEY (INDEX_ID); + +-- Table PART_COL_STATS for classes [org.apache.hadoop.hive.metastore.model.MPartitionColumnStatistics] +CREATE TABLE PART_COL_STATS +( + CS_ID bigint NOT NULL, + AVG_COL_LEN float NULL, + "COLUMN_NAME" varchar(128) NOT NULL, + COLUMN_TYPE varchar(128) NOT NULL, + DB_NAME varchar(128) NOT NULL, + BIG_DECIMAL_HIGH_VALUE varchar(255) NULL, + BIG_DECIMAL_LOW_VALUE varchar(255) NULL, + DOUBLE_HIGH_VALUE float NULL, + DOUBLE_LOW_VALUE float NULL, + LAST_ANALYZED bigint NOT NULL, + LONG_HIGH_VALUE bigint NULL, + LONG_LOW_VALUE bigint NULL, + MAX_COL_LEN bigint NULL, + NUM_DISTINCTS bigint NULL, + NUM_FALSES bigint NULL, + NUM_NULLS bigint NOT NULL, + NUM_TRUES bigint NULL, + PART_ID bigint NULL, + PARTITION_NAME varchar(767) NOT NULL, + "TABLE_NAME" varchar(128) NOT NULL +); + +ALTER TABLE PART_COL_STATS ADD CONSTRAINT PART_COL_STATS_PK PRIMARY KEY (CS_ID); + +CREATE INDEX PCS_STATS_IDX ON PART_COL_STATS (DB_NAME,TABLE_NAME,COLUMN_NAME,PARTITION_NAME); + +-- Table PART_PRIVS for classes [org.apache.hadoop.hive.metastore.model.MPartitionPrivilege] +CREATE TABLE PART_PRIVS +( + PART_GRANT_ID bigint NOT NULL, + CREATE_TIME int NOT NULL, + GRANT_OPTION smallint NOT NULL CHECK (GRANT_OPTION IN (0,1)), + GRANTOR varchar(128) NULL, + GRANTOR_TYPE varchar(128) NULL, + PART_ID bigint NULL, + PRINCIPAL_NAME varchar(128) NULL, + PRINCIPAL_TYPE varchar(128) NULL, + PART_PRIV varchar(128) NULL +); + +ALTER TABLE PART_PRIVS ADD CONSTRAINT PART_PRIVS_PK PRIMARY KEY (PART_GRANT_ID); + +-- Table SKEWED_STRING_LIST for classes [org.apache.hadoop.hive.metastore.model.MStringList] +CREATE TABLE SKEWED_STRING_LIST +( + STRING_LIST_ID bigint NOT NULL +); + +ALTER TABLE SKEWED_STRING_LIST ADD CONSTRAINT SKEWED_STRING_LIST_PK PRIMARY KEY (STRING_LIST_ID); + +-- Table ROLES for classes [org.apache.hadoop.hive.metastore.model.MRole] +CREATE TABLE ROLES +( + ROLE_ID bigint NOT NULL, + CREATE_TIME int NOT NULL, + OWNER_NAME varchar(128) NULL, + ROLE_NAME varchar(128) NULL +); + +ALTER TABLE ROLES ADD CONSTRAINT ROLES_PK PRIMARY KEY (ROLE_ID); + +-- Table PARTITIONS for classes [org.apache.hadoop.hive.metastore.model.MPartition] +CREATE TABLE PARTITIONS +( + PART_ID bigint NOT NULL, + CREATE_TIME int NOT NULL, + LAST_ACCESS_TIME int NOT NULL, + PART_NAME varchar(767) NULL, + SD_ID bigint NULL, + TBL_ID bigint NULL +); + +ALTER TABLE PARTITIONS ADD CONSTRAINT PARTITIONS_PK PRIMARY KEY (PART_ID); + +-- Table CDS for classes [org.apache.hadoop.hive.metastore.model.MColumnDescriptor] +CREATE TABLE CDS +( + CD_ID bigint NOT NULL +); + +ALTER TABLE CDS ADD CONSTRAINT CDS_PK PRIMARY KEY (CD_ID); + +-- Table VERSION for classes [org.apache.hadoop.hive.metastore.model.MVersionTable] +CREATE TABLE VERSION +( + VER_ID bigint NOT NULL, + SCHEMA_VERSION varchar(127) NOT NULL, + VERSION_COMMENT varchar(255) NOT NULL +); + +ALTER TABLE VERSION ADD CONSTRAINT VERSION_PK PRIMARY KEY (VER_ID); + +-- Table GLOBAL_PRIVS for classes [org.apache.hadoop.hive.metastore.model.MGlobalPrivilege] +CREATE TABLE GLOBAL_PRIVS +( + USER_GRANT_ID bigint NOT NULL, + CREATE_TIME int NOT NULL, + GRANT_OPTION smallint NOT NULL CHECK (GRANT_OPTION IN (0,1)), + GRANTOR varchar(128) NULL, + GRANTOR_TYPE varchar(128) NULL, + PRINCIPAL_NAME varchar(128) NULL, + PRINCIPAL_TYPE varchar(128) NULL, + USER_PRIV varchar(128) NULL +); + +ALTER TABLE GLOBAL_PRIVS ADD CONSTRAINT GLOBAL_PRIVS_PK PRIMARY KEY (USER_GRANT_ID); + +-- Table PART_COL_PRIVS for classes [org.apache.hadoop.hive.metastore.model.MPartitionColumnPrivilege] +CREATE TABLE PART_COL_PRIVS +( + PART_COLUMN_GRANT_ID bigint NOT NULL, + "COLUMN_NAME" varchar(128) NULL, + CREATE_TIME int NOT NULL, + GRANT_OPTION smallint NOT NULL CHECK (GRANT_OPTION IN (0,1)), + GRANTOR varchar(128) NULL, + GRANTOR_TYPE varchar(128) NULL, + PART_ID bigint NULL, + PRINCIPAL_NAME varchar(128) NULL, + PRINCIPAL_TYPE varchar(128) NULL, + PART_COL_PRIV varchar(128) NULL +); + +ALTER TABLE PART_COL_PRIVS ADD CONSTRAINT PART_COL_PRIVS_PK PRIMARY KEY (PART_COLUMN_GRANT_ID); + +-- Table DB_PRIVS for classes [org.apache.hadoop.hive.metastore.model.MDBPrivilege] +CREATE TABLE DB_PRIVS +( + DB_GRANT_ID bigint NOT NULL, + CREATE_TIME int NOT NULL, + DB_ID bigint NULL, + GRANT_OPTION smallint NOT NULL CHECK (GRANT_OPTION IN (0,1)), + GRANTOR varchar(128) NULL, + GRANTOR_TYPE varchar(128) NULL, + PRINCIPAL_NAME varchar(128) NULL, + PRINCIPAL_TYPE varchar(128) NULL, + DB_PRIV varchar(128) NULL +); + +ALTER TABLE DB_PRIVS ADD CONSTRAINT DB_PRIVS_PK PRIMARY KEY (DB_GRANT_ID); + +-- Table TAB_COL_STATS for classes [org.apache.hadoop.hive.metastore.model.MTableColumnStatistics] +CREATE TABLE TAB_COL_STATS +( + CS_ID bigint NOT NULL, + AVG_COL_LEN float NULL, + "COLUMN_NAME" varchar(128) NOT NULL, + COLUMN_TYPE varchar(128) NOT NULL, + DB_NAME varchar(128) NOT NULL, + BIG_DECIMAL_HIGH_VALUE varchar(255) NULL, + BIG_DECIMAL_LOW_VALUE varchar(255) NULL, + DOUBLE_HIGH_VALUE float NULL, + DOUBLE_LOW_VALUE float NULL, + LAST_ANALYZED bigint NOT NULL, + LONG_HIGH_VALUE bigint NULL, + LONG_LOW_VALUE bigint NULL, + MAX_COL_LEN bigint NULL, + NUM_DISTINCTS bigint NULL, + NUM_FALSES bigint NULL, + NUM_NULLS bigint NOT NULL, + NUM_TRUES bigint NULL, + TBL_ID bigint NULL, + "TABLE_NAME" varchar(128) NOT NULL +); + +ALTER TABLE TAB_COL_STATS ADD CONSTRAINT TAB_COL_STATS_PK PRIMARY KEY (CS_ID); + +-- Table TYPES for classes [org.apache.hadoop.hive.metastore.model.MType] +CREATE TABLE TYPES +( + TYPES_ID bigint NOT NULL, + TYPE_NAME varchar(128) NULL, + TYPE1 varchar(767) NULL, + TYPE2 varchar(767) NULL +); + +ALTER TABLE TYPES ADD CONSTRAINT TYPES_PK PRIMARY KEY (TYPES_ID); + +-- Table TBL_PRIVS for classes [org.apache.hadoop.hive.metastore.model.MTablePrivilege] +CREATE TABLE TBL_PRIVS +( + TBL_GRANT_ID bigint NOT NULL, + CREATE_TIME int NOT NULL, + GRANT_OPTION smallint NOT NULL CHECK (GRANT_OPTION IN (0,1)), + GRANTOR varchar(128) NULL, + GRANTOR_TYPE varchar(128) NULL, + PRINCIPAL_NAME varchar(128) NULL, + PRINCIPAL_TYPE varchar(128) NULL, + TBL_PRIV varchar(128) NULL, + TBL_ID bigint NULL +); + +ALTER TABLE TBL_PRIVS ADD CONSTRAINT TBL_PRIVS_PK PRIMARY KEY (TBL_GRANT_ID); + +-- Table DBS for classes [org.apache.hadoop.hive.metastore.model.MDatabase] +CREATE TABLE DBS +( + DB_ID bigint NOT NULL, + "DESC" varchar(4000) NULL, + DB_LOCATION_URI varchar(4000) NOT NULL, + "NAME" varchar(128) NULL, + OWNER_NAME varchar(128) NULL, + OWNER_TYPE varchar(10) NULL +); + +ALTER TABLE DBS ADD CONSTRAINT DBS_PK PRIMARY KEY (DB_ID); + +-- Table TBL_COL_PRIVS for classes [org.apache.hadoop.hive.metastore.model.MTableColumnPrivilege] +CREATE TABLE TBL_COL_PRIVS +( + TBL_COLUMN_GRANT_ID bigint NOT NULL, + "COLUMN_NAME" varchar(128) NULL, + CREATE_TIME int NOT NULL, + GRANT_OPTION smallint NOT NULL CHECK (GRANT_OPTION IN (0,1)), + GRANTOR varchar(128) NULL, + GRANTOR_TYPE varchar(128) NULL, + PRINCIPAL_NAME varchar(128) NULL, + PRINCIPAL_TYPE varchar(128) NULL, + TBL_COL_PRIV varchar(128) NULL, + TBL_ID bigint NULL +); + +ALTER TABLE TBL_COL_PRIVS ADD CONSTRAINT TBL_COL_PRIVS_PK PRIMARY KEY (TBL_COLUMN_GRANT_ID); + +-- Table DELEGATION_TOKENS for classes [org.apache.hadoop.hive.metastore.model.MDelegationToken] +CREATE TABLE DELEGATION_TOKENS +( + TOKEN_IDENT varchar(767) NOT NULL, + TOKEN varchar(767) NULL +); + +ALTER TABLE DELEGATION_TOKENS ADD CONSTRAINT DELEGATION_TOKENS_PK PRIMARY KEY (TOKEN_IDENT); + +-- Table SERDES for classes [org.apache.hadoop.hive.metastore.model.MSerDeInfo] +CREATE TABLE SERDES +( + SERDE_ID bigint NOT NULL, + "NAME" varchar(128) NULL, + SLIB varchar(4000) NULL +); + +ALTER TABLE SERDES ADD CONSTRAINT SERDES_PK PRIMARY KEY (SERDE_ID); + +-- Table FUNCS for classes [org.apache.hadoop.hive.metastore.model.MFunction] +CREATE TABLE FUNCS +( + FUNC_ID bigint NOT NULL, + CLASS_NAME varchar(4000) NULL, + CREATE_TIME int NOT NULL, + DB_ID bigint NULL, + FUNC_NAME varchar(128) NULL, + FUNC_TYPE int NOT NULL, + OWNER_NAME varchar(128) NULL, + OWNER_TYPE varchar(10) NULL +); + +ALTER TABLE FUNCS ADD CONSTRAINT FUNCS_PK PRIMARY KEY (FUNC_ID); + +-- Table ROLE_MAP for classes [org.apache.hadoop.hive.metastore.model.MRoleMap] +CREATE TABLE ROLE_MAP +( + ROLE_GRANT_ID bigint NOT NULL, + ADD_TIME int NOT NULL, + GRANT_OPTION smallint NOT NULL CHECK (GRANT_OPTION IN (0,1)), + GRANTOR varchar(128) NULL, + GRANTOR_TYPE varchar(128) NULL, + PRINCIPAL_NAME varchar(128) NULL, + PRINCIPAL_TYPE varchar(128) NULL, + ROLE_ID bigint NULL +); + +ALTER TABLE ROLE_MAP ADD CONSTRAINT ROLE_MAP_PK PRIMARY KEY (ROLE_GRANT_ID); + +-- Table TBLS for classes [org.apache.hadoop.hive.metastore.model.MTable] +CREATE TABLE TBLS +( + TBL_ID bigint NOT NULL, + CREATE_TIME int NOT NULL, + DB_ID bigint NULL, + LAST_ACCESS_TIME int NOT NULL, + OWNER varchar(767) NULL, + RETENTION int NOT NULL, + SD_ID bigint NULL, + TBL_NAME varchar(128) NULL, + TBL_TYPE varchar(128) NULL, + VIEW_EXPANDED_TEXT text NULL, + VIEW_ORIGINAL_TEXT text NULL +); + +ALTER TABLE TBLS ADD CONSTRAINT TBLS_PK PRIMARY KEY (TBL_ID); + +-- Table SDS for classes [org.apache.hadoop.hive.metastore.model.MStorageDescriptor] +CREATE TABLE SDS +( + SD_ID bigint NOT NULL, + CD_ID bigint NULL, + INPUT_FORMAT varchar(4000) NULL, + IS_COMPRESSED bit NOT NULL, + IS_STOREDASSUBDIRECTORIES bit NOT NULL, + LOCATION varchar(4000) NULL, + NUM_BUCKETS int NOT NULL, + OUTPUT_FORMAT varchar(4000) NULL, + SERDE_ID bigint NULL +); + +ALTER TABLE SDS ADD CONSTRAINT SDS_PK PRIMARY KEY (SD_ID); + +-- Table PARTITION_EVENTS for classes [org.apache.hadoop.hive.metastore.model.MPartitionEvent] +CREATE TABLE PARTITION_EVENTS +( + PART_NAME_ID bigint NOT NULL, + DB_NAME varchar(128) NULL, + EVENT_TIME bigint NOT NULL, + EVENT_TYPE int NOT NULL, + PARTITION_NAME varchar(767) NULL, + TBL_NAME varchar(128) NULL +); + +ALTER TABLE PARTITION_EVENTS ADD CONSTRAINT PARTITION_EVENTS_PK PRIMARY KEY (PART_NAME_ID); + +-- Table SORT_COLS for join relationship +CREATE TABLE SORT_COLS +( + SD_ID bigint NOT NULL, + "COLUMN_NAME" varchar(128) NULL, + "ORDER" int NOT NULL, + INTEGER_IDX int NOT NULL +); + +ALTER TABLE SORT_COLS ADD CONSTRAINT SORT_COLS_PK PRIMARY KEY (SD_ID,INTEGER_IDX); + +-- Table SKEWED_COL_NAMES for join relationship +CREATE TABLE SKEWED_COL_NAMES +( + SD_ID bigint NOT NULL, + SKEWED_COL_NAME varchar(255) NULL, + INTEGER_IDX int NOT NULL +); + +ALTER TABLE SKEWED_COL_NAMES ADD CONSTRAINT SKEWED_COL_NAMES_PK PRIMARY KEY (SD_ID,INTEGER_IDX); + +-- Table SKEWED_COL_VALUE_LOC_MAP for join relationship +CREATE TABLE SKEWED_COL_VALUE_LOC_MAP +( + SD_ID bigint NOT NULL, + STRING_LIST_ID_KID bigint NOT NULL, + LOCATION varchar(4000) NULL +); + +ALTER TABLE SKEWED_COL_VALUE_LOC_MAP ADD CONSTRAINT SKEWED_COL_VALUE_LOC_MAP_PK PRIMARY KEY (SD_ID,STRING_LIST_ID_KID); + +-- Table SKEWED_STRING_LIST_VALUES for join relationship +CREATE TABLE SKEWED_STRING_LIST_VALUES +( + STRING_LIST_ID bigint NOT NULL, + STRING_LIST_VALUE varchar(255) NULL, + INTEGER_IDX int NOT NULL +); + +ALTER TABLE SKEWED_STRING_LIST_VALUES ADD CONSTRAINT SKEWED_STRING_LIST_VALUES_PK PRIMARY KEY (STRING_LIST_ID,INTEGER_IDX); + +-- Table PARTITION_KEY_VALS for join relationship +CREATE TABLE PARTITION_KEY_VALS +( + PART_ID bigint NOT NULL, + PART_KEY_VAL varchar(255) NULL, + INTEGER_IDX int NOT NULL +); + +ALTER TABLE PARTITION_KEY_VALS ADD CONSTRAINT PARTITION_KEY_VALS_PK PRIMARY KEY (PART_ID,INTEGER_IDX); + +-- Table PARTITION_KEYS for join relationship +CREATE TABLE PARTITION_KEYS +( + TBL_ID bigint NOT NULL, + PKEY_COMMENT varchar(4000) NULL, + PKEY_NAME varchar(128) NOT NULL, + PKEY_TYPE varchar(767) NOT NULL, + INTEGER_IDX int NOT NULL +); + +ALTER TABLE PARTITION_KEYS ADD CONSTRAINT PARTITION_KEY_PK PRIMARY KEY (TBL_ID,PKEY_NAME); + +-- Table SKEWED_VALUES for join relationship +CREATE TABLE SKEWED_VALUES +( + SD_ID_OID bigint NOT NULL, + STRING_LIST_ID_EID bigint NULL, + INTEGER_IDX int NOT NULL +); + +ALTER TABLE SKEWED_VALUES ADD CONSTRAINT SKEWED_VALUES_PK PRIMARY KEY (SD_ID_OID,INTEGER_IDX); + +-- Table SD_PARAMS for join relationship +CREATE TABLE SD_PARAMS +( + SD_ID bigint NOT NULL, + PARAM_KEY varchar(256) NOT NULL, + PARAM_VALUE varchar(4000) NULL +); + +ALTER TABLE SD_PARAMS ADD CONSTRAINT SD_PARAMS_PK PRIMARY KEY (SD_ID,PARAM_KEY); + +-- Table FUNC_RU for join relationship +CREATE TABLE FUNC_RU +( + FUNC_ID bigint NOT NULL, + RESOURCE_TYPE int NOT NULL, + RESOURCE_URI varchar(4000) NULL, + INTEGER_IDX int NOT NULL +); + +ALTER TABLE FUNC_RU ADD CONSTRAINT FUNC_RU_PK PRIMARY KEY (FUNC_ID,INTEGER_IDX); + +-- Table TYPE_FIELDS for join relationship +CREATE TABLE TYPE_FIELDS +( + TYPE_NAME bigint NOT NULL, + COMMENT varchar(256) NULL, + FIELD_NAME varchar(128) NOT NULL, + FIELD_TYPE varchar(767) NOT NULL, + INTEGER_IDX int NOT NULL +); + +ALTER TABLE TYPE_FIELDS ADD CONSTRAINT TYPE_FIELDS_PK PRIMARY KEY (TYPE_NAME,FIELD_NAME); + +-- Table BUCKETING_COLS for join relationship +CREATE TABLE BUCKETING_COLS +( + SD_ID bigint NOT NULL, + BUCKET_COL_NAME varchar(255) NULL, + INTEGER_IDX int NOT NULL +); + +ALTER TABLE BUCKETING_COLS ADD CONSTRAINT BUCKETING_COLS_PK PRIMARY KEY (SD_ID,INTEGER_IDX); + +-- Table DATABASE_PARAMS for join relationship +CREATE TABLE DATABASE_PARAMS +( + DB_ID bigint NOT NULL, + PARAM_KEY varchar(180) NOT NULL, + PARAM_VALUE varchar(4000) NULL +); + +ALTER TABLE DATABASE_PARAMS ADD CONSTRAINT DATABASE_PARAMS_PK PRIMARY KEY (DB_ID,PARAM_KEY); + +-- Table INDEX_PARAMS for join relationship +CREATE TABLE INDEX_PARAMS +( + INDEX_ID bigint NOT NULL, + PARAM_KEY varchar(256) NOT NULL, + PARAM_VALUE varchar(4000) NULL +); + +ALTER TABLE INDEX_PARAMS ADD CONSTRAINT INDEX_PARAMS_PK PRIMARY KEY (INDEX_ID,PARAM_KEY); + +-- Table COLUMNS_V2 for join relationship +CREATE TABLE COLUMNS_V2 +( + CD_ID bigint NOT NULL, + COMMENT varchar(256) NULL, + "COLUMN_NAME" varchar(128) NOT NULL, + TYPE_NAME varchar(4000) NOT NULL, + INTEGER_IDX int NOT NULL +); + +ALTER TABLE COLUMNS_V2 ADD CONSTRAINT COLUMNS_PK PRIMARY KEY (CD_ID,"COLUMN_NAME"); + +-- Table SERDE_PARAMS for join relationship +CREATE TABLE SERDE_PARAMS +( + SERDE_ID bigint NOT NULL, + PARAM_KEY varchar(256) NOT NULL, + PARAM_VALUE varchar(4000) NULL +); + +ALTER TABLE SERDE_PARAMS ADD CONSTRAINT SERDE_PARAMS_PK PRIMARY KEY (SERDE_ID,PARAM_KEY); + +-- Table PARTITION_PARAMS for join relationship +CREATE TABLE PARTITION_PARAMS +( + PART_ID bigint NOT NULL, + PARAM_KEY varchar(256) NOT NULL, + PARAM_VALUE varchar(4000) NULL +); + +ALTER TABLE PARTITION_PARAMS ADD CONSTRAINT PARTITION_PARAMS_PK PRIMARY KEY (PART_ID,PARAM_KEY); + +-- Table TABLE_PARAMS for join relationship +CREATE TABLE TABLE_PARAMS +( + TBL_ID bigint NOT NULL, + PARAM_KEY varchar(256) NOT NULL, + PARAM_VALUE varchar(4000) NULL +); + +ALTER TABLE TABLE_PARAMS ADD CONSTRAINT TABLE_PARAMS_PK PRIMARY KEY (TBL_ID,PARAM_KEY); + +-- Constraints for table MASTER_KEYS for class(es) [org.apache.hadoop.hive.metastore.model.MMasterKey] + +-- Constraints for table IDXS for class(es) [org.apache.hadoop.hive.metastore.model.MIndex] +ALTER TABLE IDXS ADD CONSTRAINT IDXS_FK1 FOREIGN KEY (INDEX_TBL_ID) REFERENCES TBLS (TBL_ID) ; + +ALTER TABLE IDXS ADD CONSTRAINT IDXS_FK2 FOREIGN KEY (SD_ID) REFERENCES SDS (SD_ID) ; + +ALTER TABLE IDXS ADD CONSTRAINT IDXS_FK3 FOREIGN KEY (ORIG_TBL_ID) REFERENCES TBLS (TBL_ID) ; + +CREATE UNIQUE INDEX UNIQUEINDEX ON IDXS (INDEX_NAME,ORIG_TBL_ID); + +CREATE INDEX IDXS_N51 ON IDXS (SD_ID); + +CREATE INDEX IDXS_N50 ON IDXS (ORIG_TBL_ID); + +CREATE INDEX IDXS_N49 ON IDXS (INDEX_TBL_ID); + + +-- Constraints for table PART_COL_STATS for class(es) [org.apache.hadoop.hive.metastore.model.MPartitionColumnStatistics] +ALTER TABLE PART_COL_STATS ADD CONSTRAINT PART_COL_STATS_FK1 FOREIGN KEY (PART_ID) REFERENCES PARTITIONS (PART_ID) ; + +CREATE INDEX PART_COL_STATS_N49 ON PART_COL_STATS (PART_ID); + + +-- Constraints for table PART_PRIVS for class(es) [org.apache.hadoop.hive.metastore.model.MPartitionPrivilege] +ALTER TABLE PART_PRIVS ADD CONSTRAINT PART_PRIVS_FK1 FOREIGN KEY (PART_ID) REFERENCES PARTITIONS (PART_ID) ; + +CREATE INDEX PARTPRIVILEGEINDEX ON PART_PRIVS (PART_ID,PRINCIPAL_NAME,PRINCIPAL_TYPE,PART_PRIV,GRANTOR,GRANTOR_TYPE); + +CREATE INDEX PART_PRIVS_N49 ON PART_PRIVS (PART_ID); + + +-- Constraints for table SKEWED_STRING_LIST for class(es) [org.apache.hadoop.hive.metastore.model.MStringList] + +-- Constraints for table ROLES for class(es) [org.apache.hadoop.hive.metastore.model.MRole] +CREATE UNIQUE INDEX ROLEENTITYINDEX ON ROLES (ROLE_NAME); + + +-- Constraints for table PARTITIONS for class(es) [org.apache.hadoop.hive.metastore.model.MPartition] +ALTER TABLE PARTITIONS ADD CONSTRAINT PARTITIONS_FK1 FOREIGN KEY (TBL_ID) REFERENCES TBLS (TBL_ID) ; + +ALTER TABLE PARTITIONS ADD CONSTRAINT PARTITIONS_FK2 FOREIGN KEY (SD_ID) REFERENCES SDS (SD_ID) ; + +CREATE INDEX PARTITIONS_N49 ON PARTITIONS (SD_ID); + +CREATE INDEX PARTITIONS_N50 ON PARTITIONS (TBL_ID); + +CREATE UNIQUE INDEX UNIQUEPARTITION ON PARTITIONS (PART_NAME,TBL_ID); + + +-- Constraints for table CDS for class(es) [org.apache.hadoop.hive.metastore.model.MColumnDescriptor] + +-- Constraints for table VERSION for class(es) [org.apache.hadoop.hive.metastore.model.MVersionTable] + +-- Constraints for table GLOBAL_PRIVS for class(es) [org.apache.hadoop.hive.metastore.model.MGlobalPrivilege] +CREATE UNIQUE INDEX GLOBALPRIVILEGEINDEX ON GLOBAL_PRIVS (PRINCIPAL_NAME,PRINCIPAL_TYPE,USER_PRIV,GRANTOR,GRANTOR_TYPE); + + +-- Constraints for table PART_COL_PRIVS for class(es) [org.apache.hadoop.hive.metastore.model.MPartitionColumnPrivilege] +ALTER TABLE PART_COL_PRIVS ADD CONSTRAINT PART_COL_PRIVS_FK1 FOREIGN KEY (PART_ID) REFERENCES PARTITIONS (PART_ID) ; + +CREATE INDEX PART_COL_PRIVS_N49 ON PART_COL_PRIVS (PART_ID); + +CREATE INDEX PARTITIONCOLUMNPRIVILEGEINDEX ON PART_COL_PRIVS (PART_ID,"COLUMN_NAME",PRINCIPAL_NAME,PRINCIPAL_TYPE,PART_COL_PRIV,GRANTOR,GRANTOR_TYPE); + + +-- Constraints for table DB_PRIVS for class(es) [org.apache.hadoop.hive.metastore.model.MDBPrivilege] +ALTER TABLE DB_PRIVS ADD CONSTRAINT DB_PRIVS_FK1 FOREIGN KEY (DB_ID) REFERENCES DBS (DB_ID) ; + +CREATE UNIQUE INDEX DBPRIVILEGEINDEX ON DB_PRIVS (DB_ID,PRINCIPAL_NAME,PRINCIPAL_TYPE,DB_PRIV,GRANTOR,GRANTOR_TYPE); + +CREATE INDEX DB_PRIVS_N49 ON DB_PRIVS (DB_ID); + + +-- Constraints for table TAB_COL_STATS for class(es) [org.apache.hadoop.hive.metastore.model.MTableColumnStatistics] +ALTER TABLE TAB_COL_STATS ADD CONSTRAINT TAB_COL_STATS_FK1 FOREIGN KEY (TBL_ID) REFERENCES TBLS (TBL_ID) ; + +CREATE INDEX TAB_COL_STATS_N49 ON TAB_COL_STATS (TBL_ID); + + +-- Constraints for table TYPES for class(es) [org.apache.hadoop.hive.metastore.model.MType] +CREATE UNIQUE INDEX UNIQUETYPE ON TYPES (TYPE_NAME); + + +-- Constraints for table TBL_PRIVS for class(es) [org.apache.hadoop.hive.metastore.model.MTablePrivilege] +ALTER TABLE TBL_PRIVS ADD CONSTRAINT TBL_PRIVS_FK1 FOREIGN KEY (TBL_ID) REFERENCES TBLS (TBL_ID) ; + +CREATE INDEX TBL_PRIVS_N49 ON TBL_PRIVS (TBL_ID); + +CREATE INDEX TABLEPRIVILEGEINDEX ON TBL_PRIVS (TBL_ID,PRINCIPAL_NAME,PRINCIPAL_TYPE,TBL_PRIV,GRANTOR,GRANTOR_TYPE); + + +-- Constraints for table DBS for class(es) [org.apache.hadoop.hive.metastore.model.MDatabase] +CREATE UNIQUE INDEX UNIQUEDATABASE ON DBS ("NAME"); + + +-- Constraints for table TBL_COL_PRIVS for class(es) [org.apache.hadoop.hive.metastore.model.MTableColumnPrivilege] +ALTER TABLE TBL_COL_PRIVS ADD CONSTRAINT TBL_COL_PRIVS_FK1 FOREIGN KEY (TBL_ID) REFERENCES TBLS (TBL_ID) ; + +CREATE INDEX TABLECOLUMNPRIVILEGEINDEX ON TBL_COL_PRIVS (TBL_ID,"COLUMN_NAME",PRINCIPAL_NAME,PRINCIPAL_TYPE,TBL_COL_PRIV,GRANTOR,GRANTOR_TYPE); + +CREATE INDEX TBL_COL_PRIVS_N49 ON TBL_COL_PRIVS (TBL_ID); + + +-- Constraints for table DELEGATION_TOKENS for class(es) [org.apache.hadoop.hive.metastore.model.MDelegationToken] + +-- Constraints for table SERDES for class(es) [org.apache.hadoop.hive.metastore.model.MSerDeInfo] + +-- Constraints for table FUNCS for class(es) [org.apache.hadoop.hive.metastore.model.MFunction] +ALTER TABLE FUNCS ADD CONSTRAINT FUNCS_FK1 FOREIGN KEY (DB_ID) REFERENCES DBS (DB_ID) ; + +CREATE UNIQUE INDEX UNIQUEFUNCTION ON FUNCS (FUNC_NAME,DB_ID); + +CREATE INDEX FUNCS_N49 ON FUNCS (DB_ID); + + +-- Constraints for table ROLE_MAP for class(es) [org.apache.hadoop.hive.metastore.model.MRoleMap] +ALTER TABLE ROLE_MAP ADD CONSTRAINT ROLE_MAP_FK1 FOREIGN KEY (ROLE_ID) REFERENCES ROLES (ROLE_ID) ; + +CREATE INDEX ROLE_MAP_N49 ON ROLE_MAP (ROLE_ID); + +CREATE UNIQUE INDEX USERROLEMAPINDEX ON ROLE_MAP (PRINCIPAL_NAME,ROLE_ID,GRANTOR,GRANTOR_TYPE); + + +-- Constraints for table TBLS for class(es) [org.apache.hadoop.hive.metastore.model.MTable] +ALTER TABLE TBLS ADD CONSTRAINT TBLS_FK2 FOREIGN KEY (SD_ID) REFERENCES SDS (SD_ID) ; + +ALTER TABLE TBLS ADD CONSTRAINT TBLS_FK1 FOREIGN KEY (DB_ID) REFERENCES DBS (DB_ID) ; + +CREATE INDEX TBLS_N50 ON TBLS (SD_ID); + +CREATE UNIQUE INDEX UNIQUETABLE ON TBLS (TBL_NAME,DB_ID); + +CREATE INDEX TBLS_N49 ON TBLS (DB_ID); + + +-- Constraints for table SDS for class(es) [org.apache.hadoop.hive.metastore.model.MStorageDescriptor] +ALTER TABLE SDS ADD CONSTRAINT SDS_FK1 FOREIGN KEY (SERDE_ID) REFERENCES SERDES (SERDE_ID) ; + +ALTER TABLE SDS ADD CONSTRAINT SDS_FK2 FOREIGN KEY (CD_ID) REFERENCES CDS (CD_ID) ; + +CREATE INDEX SDS_N50 ON SDS (CD_ID); + +CREATE INDEX SDS_N49 ON SDS (SERDE_ID); + + +-- Constraints for table PARTITION_EVENTS for class(es) [org.apache.hadoop.hive.metastore.model.MPartitionEvent] +CREATE INDEX PARTITIONEVENTINDEX ON PARTITION_EVENTS (PARTITION_NAME); + + +-- Constraints for table SORT_COLS +ALTER TABLE SORT_COLS ADD CONSTRAINT SORT_COLS_FK1 FOREIGN KEY (SD_ID) REFERENCES SDS (SD_ID) ; + +CREATE INDEX SORT_COLS_N49 ON SORT_COLS (SD_ID); + + +-- Constraints for table SKEWED_COL_NAMES +ALTER TABLE SKEWED_COL_NAMES ADD CONSTRAINT SKEWED_COL_NAMES_FK1 FOREIGN KEY (SD_ID) REFERENCES SDS (SD_ID) ; + +CREATE INDEX SKEWED_COL_NAMES_N49 ON SKEWED_COL_NAMES (SD_ID); + + +-- Constraints for table SKEWED_COL_VALUE_LOC_MAP +ALTER TABLE SKEWED_COL_VALUE_LOC_MAP ADD CONSTRAINT SKEWED_COL_VALUE_LOC_MAP_FK1 FOREIGN KEY (SD_ID) REFERENCES SDS (SD_ID) ; + +ALTER TABLE SKEWED_COL_VALUE_LOC_MAP ADD CONSTRAINT SKEWED_COL_VALUE_LOC_MAP_FK2 FOREIGN KEY (STRING_LIST_ID_KID) REFERENCES SKEWED_STRING_LIST (STRING_LIST_ID) ; + +CREATE INDEX SKEWED_COL_VALUE_LOC_MAP_N50 ON SKEWED_COL_VALUE_LOC_MAP (STRING_LIST_ID_KID); + +CREATE INDEX SKEWED_COL_VALUE_LOC_MAP_N49 ON SKEWED_COL_VALUE_LOC_MAP (SD_ID); + + +-- Constraints for table SKEWED_STRING_LIST_VALUES +ALTER TABLE SKEWED_STRING_LIST_VALUES ADD CONSTRAINT SKEWED_STRING_LIST_VALUES_FK1 FOREIGN KEY (STRING_LIST_ID) REFERENCES SKEWED_STRING_LIST (STRING_LIST_ID) ; + +CREATE INDEX SKEWED_STRING_LIST_VALUES_N49 ON SKEWED_STRING_LIST_VALUES (STRING_LIST_ID); + + +-- Constraints for table PARTITION_KEY_VALS +ALTER TABLE PARTITION_KEY_VALS ADD CONSTRAINT PARTITION_KEY_VALS_FK1 FOREIGN KEY (PART_ID) REFERENCES PARTITIONS (PART_ID) ; + +CREATE INDEX PARTITION_KEY_VALS_N49 ON PARTITION_KEY_VALS (PART_ID); + + +-- Constraints for table PARTITION_KEYS +ALTER TABLE PARTITION_KEYS ADD CONSTRAINT PARTITION_KEYS_FK1 FOREIGN KEY (TBL_ID) REFERENCES TBLS (TBL_ID) ; + +CREATE INDEX PARTITION_KEYS_N49 ON PARTITION_KEYS (TBL_ID); + + +-- Constraints for table SKEWED_VALUES +ALTER TABLE SKEWED_VALUES ADD CONSTRAINT SKEWED_VALUES_FK1 FOREIGN KEY (SD_ID_OID) REFERENCES SDS (SD_ID) ; + +ALTER TABLE SKEWED_VALUES ADD CONSTRAINT SKEWED_VALUES_FK2 FOREIGN KEY (STRING_LIST_ID_EID) REFERENCES SKEWED_STRING_LIST (STRING_LIST_ID) ; + +CREATE INDEX SKEWED_VALUES_N50 ON SKEWED_VALUES (SD_ID_OID); + +CREATE INDEX SKEWED_VALUES_N49 ON SKEWED_VALUES (STRING_LIST_ID_EID); + + +-- Constraints for table SD_PARAMS +ALTER TABLE SD_PARAMS ADD CONSTRAINT SD_PARAMS_FK1 FOREIGN KEY (SD_ID) REFERENCES SDS (SD_ID) ; + +CREATE INDEX SD_PARAMS_N49 ON SD_PARAMS (SD_ID); + + +-- Constraints for table FUNC_RU +ALTER TABLE FUNC_RU ADD CONSTRAINT FUNC_RU_FK1 FOREIGN KEY (FUNC_ID) REFERENCES FUNCS (FUNC_ID) ; + +CREATE INDEX FUNC_RU_N49 ON FUNC_RU (FUNC_ID); + + +-- Constraints for table TYPE_FIELDS +ALTER TABLE TYPE_FIELDS ADD CONSTRAINT TYPE_FIELDS_FK1 FOREIGN KEY (TYPE_NAME) REFERENCES TYPES (TYPES_ID) ; + +CREATE INDEX TYPE_FIELDS_N49 ON TYPE_FIELDS (TYPE_NAME); + + +-- Constraints for table BUCKETING_COLS +ALTER TABLE BUCKETING_COLS ADD CONSTRAINT BUCKETING_COLS_FK1 FOREIGN KEY (SD_ID) REFERENCES SDS (SD_ID) ; + +CREATE INDEX BUCKETING_COLS_N49 ON BUCKETING_COLS (SD_ID); + + +-- Constraints for table DATABASE_PARAMS +ALTER TABLE DATABASE_PARAMS ADD CONSTRAINT DATABASE_PARAMS_FK1 FOREIGN KEY (DB_ID) REFERENCES DBS (DB_ID) ; + +CREATE INDEX DATABASE_PARAMS_N49 ON DATABASE_PARAMS (DB_ID); + + +-- Constraints for table INDEX_PARAMS +ALTER TABLE INDEX_PARAMS ADD CONSTRAINT INDEX_PARAMS_FK1 FOREIGN KEY (INDEX_ID) REFERENCES IDXS (INDEX_ID) ; + +CREATE INDEX INDEX_PARAMS_N49 ON INDEX_PARAMS (INDEX_ID); + + +-- Constraints for table COLUMNS_V2 +ALTER TABLE COLUMNS_V2 ADD CONSTRAINT COLUMNS_V2_FK1 FOREIGN KEY (CD_ID) REFERENCES CDS (CD_ID) ; + +CREATE INDEX COLUMNS_V2_N49 ON COLUMNS_V2 (CD_ID); + + +-- Constraints for table SERDE_PARAMS +ALTER TABLE SERDE_PARAMS ADD CONSTRAINT SERDE_PARAMS_FK1 FOREIGN KEY (SERDE_ID) REFERENCES SERDES (SERDE_ID) ; + +CREATE INDEX SERDE_PARAMS_N49 ON SERDE_PARAMS (SERDE_ID); + + +-- Constraints for table PARTITION_PARAMS +ALTER TABLE PARTITION_PARAMS ADD CONSTRAINT PARTITION_PARAMS_FK1 FOREIGN KEY (PART_ID) REFERENCES PARTITIONS (PART_ID) ; + +CREATE INDEX PARTITION_PARAMS_N49 ON PARTITION_PARAMS (PART_ID); + + +-- Constraints for table TABLE_PARAMS +ALTER TABLE TABLE_PARAMS ADD CONSTRAINT TABLE_PARAMS_FK1 FOREIGN KEY (TBL_ID) REFERENCES TBLS (TBL_ID) ; + +CREATE INDEX TABLE_PARAMS_N49 ON TABLE_PARAMS (TBL_ID); + + + +-- ----------------------------------------------------------------------------------------------------------------------------------------------- +-- Transaction and Lock Tables +-- These are not part of package jdo, so if you are going to regenerate this file you need to manually add the following section back to the file. +-- ----------------------------------------------------------------------------------------------------------------------------------------------- +CREATE TABLE COMPACTION_QUEUE( + CQ_ID bigint NOT NULL, + CQ_DATABASE varchar(128) NOT NULL, + CQ_TABLE varchar(128) NOT NULL, + CQ_PARTITION varchar(767) NULL, + CQ_STATE char(1) NOT NULL, + CQ_TYPE char(1) NOT NULL, + CQ_WORKER_ID varchar(128) NULL, + CQ_START bigint NULL, + CQ_RUN_AS varchar(128) NULL, +PRIMARY KEY CLUSTERED +( + CQ_ID ASC +) +); + +CREATE TABLE COMPLETED_TXN_COMPONENTS( + CTC_TXNID bigint NULL, + CTC_DATABASE varchar(128) NOT NULL, + CTC_TABLE varchar(128) NULL, + CTC_PARTITION varchar(767) NULL +); + +CREATE TABLE HIVE_LOCKS( + HL_LOCK_EXT_ID bigint NOT NULL, + HL_LOCK_INT_ID bigint NOT NULL, + HL_TXNID bigint NULL, + HL_DB varchar(128) NOT NULL, + HL_TABLE varchar(128) NULL, + HL_PARTITION varchar(767) NULL, + HL_LOCK_STATE char(1) NOT NULL, + HL_LOCK_TYPE char(1) NOT NULL, + HL_LAST_HEARTBEAT bigint NOT NULL, + HL_ACQUIRED_AT bigint NULL, + HL_USER varchar(128) NOT NULL, + HL_HOST varchar(128) NOT NULL, +PRIMARY KEY CLUSTERED +( + HL_LOCK_EXT_ID ASC, + HL_LOCK_INT_ID ASC +) +); + +CREATE TABLE NEXT_COMPACTION_QUEUE_ID( + NCQ_NEXT bigint NOT NULL +); + +INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1); + +CREATE TABLE NEXT_LOCK_ID( + NL_NEXT bigint NOT NULL +); + +INSERT INTO NEXT_LOCK_ID VALUES(1); + +CREATE TABLE NEXT_TXN_ID( + NTXN_NEXT bigint NOT NULL +); + +INSERT INTO NEXT_TXN_ID VALUES(1); + +CREATE TABLE TXNS( + TXN_ID bigint NOT NULL, + TXN_STATE char(1) NOT NULL, + TXN_STARTED bigint NOT NULL, + TXN_LAST_HEARTBEAT bigint NOT NULL, + TXN_USER varchar(128) NOT NULL, + TXN_HOST varchar(128) NOT NULL, +PRIMARY KEY CLUSTERED +( + TXN_ID ASC +) +); + +CREATE TABLE TXN_COMPONENTS( + TC_TXNID bigint NULL, + TC_DATABASE varchar(128) NOT NULL, + TC_TABLE varchar(128) NULL, + TC_PARTITION varchar(767) NULL +); + +ALTER TABLE TXN_COMPONENTS WITH CHECK ADD FOREIGN KEY(TC_TXNID) REFERENCES TXNS (TXN_ID); + + + +-- ----------------------------------------------------------------- +-- Record schema version. Should be the last step in the init script +-- ----------------------------------------------------------------- +INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES (1, '0.15.0', 'Hive release version 0.15.0'); diff --git a/metastore/scripts/upgrade/mssql/hive-txn-schema-0.14.0.mssql.sql b/metastore/scripts/upgrade/mssql/hive-txn-schema-0.14.0.mssql.sql new file mode 100644 index 0000000..b6e32ef --- /dev/null +++ b/metastore/scripts/upgrade/mssql/hive-txn-schema-0.14.0.mssql.sql @@ -0,0 +1,101 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the License); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an AS IS BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- +-- Tables for transaction management +-- + +CREATE TABLE COMPACTION_QUEUE( + CQ_ID int NOT NULL, + CQ_DATABASE varchar(128) NOT NULL, + CQ_TABLE varchar(128) NOT NULL, + CQ_PARTITION varchar(767) NULL, + CQ_STATE char(1) NOT NULL, + CQ_TYPE char(1) NOT NULL, + CQ_WORKER_ID varchar(128) NULL, + CQ_START int NULL, + CQ_RUN_AS varchar(128) NULL, +PRIMARY KEY CLUSTERED +( + CQ_ID ASC +) +); + +CREATE TABLE COMPLETED_TXN_COMPONENTS( + CTC_TXNID int NULL, + CTC_DATABASE varchar(128) NOT NULL, + CTC_TABLE varchar(128) NULL, + CTC_PARTITION varchar(767) NULL +); + +CREATE TABLE HIVE_LOCKS( + HL_LOCK_EXT_ID int NOT NULL, + HL_LOCK_INT_ID int NOT NULL, + HL_TXNID int NULL, + HL_DB varchar(128) NOT NULL, + HL_TABLE varchar(128) NULL, + HL_PARTITION varchar(767) NULL, + HL_LOCK_STATE char(1) NOT NULL, + HL_LOCK_TYPE char(1) NOT NULL, + HL_LAST_HEARTBEAT int NOT NULL, + HL_ACQUIRED_AT int NULL, + HL_USER varchar(128) NOT NULL, + HL_HOST varchar(128) NOT NULL, +PRIMARY KEY CLUSTERED +( + HL_LOCK_EXT_ID ASC, + HL_LOCK_INT_ID ASC +) +); + +CREATE TABLE NEXT_COMPACTION_QUEUE_ID( + NCQ_NEXT int NOT NULL +); + +INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1); + +CREATE TABLE NEXT_LOCK_ID( + NL_NEXT int NOT NULL +); + +INSERT INTO NEXT_LOCK_ID VALUES(1); + +CREATE TABLE NEXT_TXN_ID( + NTXN_NEXT int NOT NULL +); + +INSERT INTO NEXT_TXN_ID VALUES(1); + +CREATE TABLE TXNS( + TXN_ID int NOT NULL, + TXN_STATE char(1) NOT NULL, + TXN_STARTED int NOT NULL, + TXN_LAST_HEARTBEAT int NOT NULL, + TXN_USER varchar(128) NOT NULL, + TXN_HOST varchar(128) NOT NULL, +PRIMARY KEY CLUSTERED +( + TXN_ID ASC +) +); + +CREATE TABLE TXN_COMPONENTS( + TC_TXNID int NULL, + TC_DATABASE varchar(128) NOT NULL, + TC_TABLE varchar(128) NULL, + TC_PARTITION varchar(767) NULL +); + +ALTER TABLE TXN_COMPONENTS WITH CHECK ADD FOREIGN KEY(TC_TXNID) REFERENCES TXNS (TXN_ID); diff --git a/metastore/scripts/upgrade/mssql/upgrade-0.14.0-to-0.15.0.mssql.sql b/metastore/scripts/upgrade/mssql/upgrade-0.14.0-to-0.15.0.mssql.sql new file mode 100644 index 0000000..81a27d0 --- /dev/null +++ b/metastore/scripts/upgrade/mssql/upgrade-0.14.0-to-0.15.0.mssql.sql @@ -0,0 +1,4 @@ +SELECT 'Upgrading MetaStore schema from 0.14.0 to 0.15.0' AS MESSAGE; + +UPDATE VERSION SET SCHEMA_VERSION='0.15.0', VERSION_COMMENT='Hive release version 0.15.0' where VER_ID=1; +SELECT 'Finished upgrading MetaStore schema from 0.14.0 to 0.15.0' AS MESSAGE; diff --git a/metastore/scripts/upgrade/mssql/upgrade.order.mssql b/metastore/scripts/upgrade/mssql/upgrade.order.mssql index 638345a..8745575 100644 --- a/metastore/scripts/upgrade/mssql/upgrade.order.mssql +++ b/metastore/scripts/upgrade/mssql/upgrade.order.mssql @@ -1,3 +1,4 @@ 0.11.0-to-0.12.0 0.12.0-to-0.13.0 0.13.0-to-0.14.0 +0.14.0-to-0.15.0 diff --git a/metastore/scripts/upgrade/mysql/hive-schema-0.14.0.mysql.sql b/metastore/scripts/upgrade/mysql/hive-schema-0.14.0.mysql.sql index b479aa2..e44e7c2 100644 --- a/metastore/scripts/upgrade/mysql/hive-schema-0.14.0.mysql.sql +++ b/metastore/scripts/upgrade/mysql/hive-schema-0.14.0.mysql.sql @@ -798,8 +798,77 @@ CREATE TABLE IF NOT EXISTS `FUNC_RU` ( -- ---------------------------- -- Transaction and Lock Tables +-- These are not part of package jdo, so if you are going to regenerate this file you need to manually add the following section back to the file. -- ---------------------------- -SOURCE hive-txn-schema-0.13.0.mysql.sql; +CREATE TABLE TXNS ( + TXN_ID bigint PRIMARY KEY, + TXN_STATE char(1) NOT NULL, + TXN_STARTED bigint NOT NULL, + TXN_LAST_HEARTBEAT bigint NOT NULL, + TXN_USER varchar(128) NOT NULL, + TXN_HOST varchar(128) NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE TABLE TXN_COMPONENTS ( + TC_TXNID bigint, + TC_DATABASE varchar(128) NOT NULL, + TC_TABLE varchar(128), + TC_PARTITION varchar(767), + FOREIGN KEY (TC_TXNID) REFERENCES TXNS (TXN_ID) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE TABLE COMPLETED_TXN_COMPONENTS ( + CTC_TXNID bigint, + CTC_DATABASE varchar(128) NOT NULL, + CTC_TABLE varchar(128), + CTC_PARTITION varchar(767) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE TABLE NEXT_TXN_ID ( + NTXN_NEXT bigint NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +INSERT INTO NEXT_TXN_ID VALUES(1); + +CREATE TABLE HIVE_LOCKS ( + HL_LOCK_EXT_ID bigint NOT NULL, + HL_LOCK_INT_ID bigint NOT NULL, + HL_TXNID bigint, + HL_DB varchar(128) NOT NULL, + HL_TABLE varchar(128), + HL_PARTITION varchar(767), + HL_LOCK_STATE char(1) not null, + HL_LOCK_TYPE char(1) not null, + HL_LAST_HEARTBEAT bigint NOT NULL, + HL_ACQUIRED_AT bigint, + HL_USER varchar(128) NOT NULL, + HL_HOST varchar(128) NOT NULL, + PRIMARY KEY(HL_LOCK_EXT_ID, HL_LOCK_INT_ID), + KEY HIVE_LOCK_TXNID_INDEX (HL_TXNID) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE INDEX HL_TXNID_IDX ON HIVE_LOCKS (HL_TXNID); + +CREATE TABLE NEXT_LOCK_ID ( + NL_NEXT bigint NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +INSERT INTO NEXT_LOCK_ID VALUES(1); + +CREATE TABLE COMPACTION_QUEUE ( + CQ_ID bigint PRIMARY KEY, + CQ_DATABASE varchar(128) NOT NULL, + CQ_TABLE varchar(128) NOT NULL, + CQ_PARTITION varchar(767), + CQ_STATE char(1) NOT NULL, + CQ_TYPE char(1) NOT NULL, + CQ_WORKER_ID varchar(128), + CQ_START bigint, + CQ_RUN_AS varchar(128) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE TABLE NEXT_COMPACTION_QUEUE_ID ( + NCQ_NEXT bigint NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1); -- ----------------------------------------------------------------- -- Record schema version. Should be the last step in the init script diff --git a/metastore/scripts/upgrade/mysql/hive-schema-0.15.0.mysql.sql b/metastore/scripts/upgrade/mysql/hive-schema-0.15.0.mysql.sql new file mode 100644 index 0000000..a811bb6 --- /dev/null +++ b/metastore/scripts/upgrade/mysql/hive-schema-0.15.0.mysql.sql @@ -0,0 +1,820 @@ +-- MySQL dump 10.13 Distrib 5.5.25, for osx10.6 (i386) +-- +-- Host: localhost Database: test +-- ------------------------------------------------------ +-- Server version 5.5.25 + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + +-- +-- Table structure for table `BUCKETING_COLS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `BUCKETING_COLS` ( + `SD_ID` bigint(20) NOT NULL, + `BUCKET_COL_NAME` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `INTEGER_IDX` int(11) NOT NULL, + PRIMARY KEY (`SD_ID`,`INTEGER_IDX`), + KEY `BUCKETING_COLS_N49` (`SD_ID`), + CONSTRAINT `BUCKETING_COLS_FK1` FOREIGN KEY (`SD_ID`) REFERENCES `SDS` (`SD_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `CDS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `CDS` ( + `CD_ID` bigint(20) NOT NULL, + PRIMARY KEY (`CD_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `COLUMNS_V2` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `COLUMNS_V2` ( + `CD_ID` bigint(20) NOT NULL, + `COMMENT` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `COLUMN_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `TYPE_NAME` varchar(4000) DEFAULT NULL, + `INTEGER_IDX` int(11) NOT NULL, + PRIMARY KEY (`CD_ID`,`COLUMN_NAME`), + KEY `COLUMNS_V2_N49` (`CD_ID`), + CONSTRAINT `COLUMNS_V2_FK1` FOREIGN KEY (`CD_ID`) REFERENCES `CDS` (`CD_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `DATABASE_PARAMS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `DATABASE_PARAMS` ( + `DB_ID` bigint(20) NOT NULL, + `PARAM_KEY` varchar(180) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `PARAM_VALUE` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + PRIMARY KEY (`DB_ID`,`PARAM_KEY`), + KEY `DATABASE_PARAMS_N49` (`DB_ID`), + CONSTRAINT `DATABASE_PARAMS_FK1` FOREIGN KEY (`DB_ID`) REFERENCES `DBS` (`DB_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `DBS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `DBS` ( + `DB_ID` bigint(20) NOT NULL, + `DESC` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `DB_LOCATION_URI` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `OWNER_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `OWNER_TYPE` varchar(10) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + PRIMARY KEY (`DB_ID`), + UNIQUE KEY `UNIQUE_DATABASE` (`NAME`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `DB_PRIVS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `DB_PRIVS` ( + `DB_GRANT_ID` bigint(20) NOT NULL, + `CREATE_TIME` int(11) NOT NULL, + `DB_ID` bigint(20) DEFAULT NULL, + `GRANT_OPTION` smallint(6) NOT NULL, + `GRANTOR` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `GRANTOR_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `PRINCIPAL_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `PRINCIPAL_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `DB_PRIV` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + PRIMARY KEY (`DB_GRANT_ID`), + UNIQUE KEY `DBPRIVILEGEINDEX` (`DB_ID`,`PRINCIPAL_NAME`,`PRINCIPAL_TYPE`,`DB_PRIV`,`GRANTOR`,`GRANTOR_TYPE`), + KEY `DB_PRIVS_N49` (`DB_ID`), + CONSTRAINT `DB_PRIVS_FK1` FOREIGN KEY (`DB_ID`) REFERENCES `DBS` (`DB_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `GLOBAL_PRIVS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `GLOBAL_PRIVS` ( + `USER_GRANT_ID` bigint(20) NOT NULL, + `CREATE_TIME` int(11) NOT NULL, + `GRANT_OPTION` smallint(6) NOT NULL, + `GRANTOR` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `GRANTOR_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `PRINCIPAL_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `PRINCIPAL_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `USER_PRIV` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + PRIMARY KEY (`USER_GRANT_ID`), + UNIQUE KEY `GLOBALPRIVILEGEINDEX` (`PRINCIPAL_NAME`,`PRINCIPAL_TYPE`,`USER_PRIV`,`GRANTOR`,`GRANTOR_TYPE`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `IDXS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `IDXS` ( + `INDEX_ID` bigint(20) NOT NULL, + `CREATE_TIME` int(11) NOT NULL, + `DEFERRED_REBUILD` bit(1) NOT NULL, + `INDEX_HANDLER_CLASS` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `INDEX_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `INDEX_TBL_ID` bigint(20) DEFAULT NULL, + `LAST_ACCESS_TIME` int(11) NOT NULL, + `ORIG_TBL_ID` bigint(20) DEFAULT NULL, + `SD_ID` bigint(20) DEFAULT NULL, + PRIMARY KEY (`INDEX_ID`), + UNIQUE KEY `UNIQUEINDEX` (`INDEX_NAME`,`ORIG_TBL_ID`), + KEY `IDXS_N51` (`SD_ID`), + KEY `IDXS_N50` (`INDEX_TBL_ID`), + KEY `IDXS_N49` (`ORIG_TBL_ID`), + CONSTRAINT `IDXS_FK1` FOREIGN KEY (`ORIG_TBL_ID`) REFERENCES `TBLS` (`TBL_ID`), + CONSTRAINT `IDXS_FK2` FOREIGN KEY (`SD_ID`) REFERENCES `SDS` (`SD_ID`), + CONSTRAINT `IDXS_FK3` FOREIGN KEY (`INDEX_TBL_ID`) REFERENCES `TBLS` (`TBL_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `INDEX_PARAMS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `INDEX_PARAMS` ( + `INDEX_ID` bigint(20) NOT NULL, + `PARAM_KEY` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `PARAM_VALUE` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + PRIMARY KEY (`INDEX_ID`,`PARAM_KEY`), + KEY `INDEX_PARAMS_N49` (`INDEX_ID`), + CONSTRAINT `INDEX_PARAMS_FK1` FOREIGN KEY (`INDEX_ID`) REFERENCES `IDXS` (`INDEX_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `NUCLEUS_TABLES` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `NUCLEUS_TABLES` ( + `CLASS_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `TABLE_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `TYPE` varchar(4) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `OWNER` varchar(2) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `VERSION` varchar(20) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `INTERFACE_NAME` varchar(255) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + PRIMARY KEY (`CLASS_NAME`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `PARTITIONS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `PARTITIONS` ( + `PART_ID` bigint(20) NOT NULL, + `CREATE_TIME` int(11) NOT NULL, + `LAST_ACCESS_TIME` int(11) NOT NULL, + `PART_NAME` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `SD_ID` bigint(20) DEFAULT NULL, + `TBL_ID` bigint(20) DEFAULT NULL, + `LINK_TARGET_ID` bigint(20) DEFAULT NULL, + PRIMARY KEY (`PART_ID`), + UNIQUE KEY `UNIQUEPARTITION` (`PART_NAME`,`TBL_ID`), + KEY `PARTITIONS_N49` (`TBL_ID`), + KEY `PARTITIONS_N50` (`SD_ID`), + KEY `PARTITIONS_N51` (`LINK_TARGET_ID`), + CONSTRAINT `PARTITIONS_FK1` FOREIGN KEY (`TBL_ID`) REFERENCES `TBLS` (`TBL_ID`), + CONSTRAINT `PARTITIONS_FK2` FOREIGN KEY (`SD_ID`) REFERENCES `SDS` (`SD_ID`), + CONSTRAINT `PARTITIONS_FK3` FOREIGN KEY (`LINK_TARGET_ID`) REFERENCES `PARTITIONS` (`PART_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `PARTITION_EVENTS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `PARTITION_EVENTS` ( + `PART_NAME_ID` bigint(20) NOT NULL, + `DB_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `EVENT_TIME` bigint(20) NOT NULL, + `EVENT_TYPE` int(11) NOT NULL, + `PARTITION_NAME` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `TBL_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + PRIMARY KEY (`PART_NAME_ID`), + KEY `PARTITIONEVENTINDEX` (`PARTITION_NAME`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `PARTITION_KEYS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `PARTITION_KEYS` ( + `TBL_ID` bigint(20) NOT NULL, + `PKEY_COMMENT` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `PKEY_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `PKEY_TYPE` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `INTEGER_IDX` int(11) NOT NULL, + PRIMARY KEY (`TBL_ID`,`PKEY_NAME`), + KEY `PARTITION_KEYS_N49` (`TBL_ID`), + CONSTRAINT `PARTITION_KEYS_FK1` FOREIGN KEY (`TBL_ID`) REFERENCES `TBLS` (`TBL_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `PARTITION_KEY_VALS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `PARTITION_KEY_VALS` ( + `PART_ID` bigint(20) NOT NULL, + `PART_KEY_VAL` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `INTEGER_IDX` int(11) NOT NULL, + PRIMARY KEY (`PART_ID`,`INTEGER_IDX`), + KEY `PARTITION_KEY_VALS_N49` (`PART_ID`), + CONSTRAINT `PARTITION_KEY_VALS_FK1` FOREIGN KEY (`PART_ID`) REFERENCES `PARTITIONS` (`PART_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `PARTITION_PARAMS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `PARTITION_PARAMS` ( + `PART_ID` bigint(20) NOT NULL, + `PARAM_KEY` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `PARAM_VALUE` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + PRIMARY KEY (`PART_ID`,`PARAM_KEY`), + KEY `PARTITION_PARAMS_N49` (`PART_ID`), + CONSTRAINT `PARTITION_PARAMS_FK1` FOREIGN KEY (`PART_ID`) REFERENCES `PARTITIONS` (`PART_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `PART_COL_PRIVS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `PART_COL_PRIVS` ( + `PART_COLUMN_GRANT_ID` bigint(20) NOT NULL, + `COLUMN_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `CREATE_TIME` int(11) NOT NULL, + `GRANT_OPTION` smallint(6) NOT NULL, + `GRANTOR` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `GRANTOR_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `PART_ID` bigint(20) DEFAULT NULL, + `PRINCIPAL_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `PRINCIPAL_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `PART_COL_PRIV` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + PRIMARY KEY (`PART_COLUMN_GRANT_ID`), + KEY `PART_COL_PRIVS_N49` (`PART_ID`), + KEY `PARTITIONCOLUMNPRIVILEGEINDEX` (`PART_ID`,`COLUMN_NAME`,`PRINCIPAL_NAME`,`PRINCIPAL_TYPE`,`PART_COL_PRIV`,`GRANTOR`,`GRANTOR_TYPE`), + CONSTRAINT `PART_COL_PRIVS_FK1` FOREIGN KEY (`PART_ID`) REFERENCES `PARTITIONS` (`PART_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `PART_PRIVS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `PART_PRIVS` ( + `PART_GRANT_ID` bigint(20) NOT NULL, + `CREATE_TIME` int(11) NOT NULL, + `GRANT_OPTION` smallint(6) NOT NULL, + `GRANTOR` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `GRANTOR_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `PART_ID` bigint(20) DEFAULT NULL, + `PRINCIPAL_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `PRINCIPAL_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `PART_PRIV` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + PRIMARY KEY (`PART_GRANT_ID`), + KEY `PARTPRIVILEGEINDEX` (`PART_ID`,`PRINCIPAL_NAME`,`PRINCIPAL_TYPE`,`PART_PRIV`,`GRANTOR`,`GRANTOR_TYPE`), + KEY `PART_PRIVS_N49` (`PART_ID`), + CONSTRAINT `PART_PRIVS_FK1` FOREIGN KEY (`PART_ID`) REFERENCES `PARTITIONS` (`PART_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `ROLES` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `ROLES` ( + `ROLE_ID` bigint(20) NOT NULL, + `CREATE_TIME` int(11) NOT NULL, + `OWNER_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `ROLE_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + PRIMARY KEY (`ROLE_ID`), + UNIQUE KEY `ROLEENTITYINDEX` (`ROLE_NAME`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `ROLE_MAP` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `ROLE_MAP` ( + `ROLE_GRANT_ID` bigint(20) NOT NULL, + `ADD_TIME` int(11) NOT NULL, + `GRANT_OPTION` smallint(6) NOT NULL, + `GRANTOR` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `GRANTOR_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `PRINCIPAL_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `PRINCIPAL_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `ROLE_ID` bigint(20) DEFAULT NULL, + PRIMARY KEY (`ROLE_GRANT_ID`), + UNIQUE KEY `USERROLEMAPINDEX` (`PRINCIPAL_NAME`,`ROLE_ID`,`GRANTOR`,`GRANTOR_TYPE`), + KEY `ROLE_MAP_N49` (`ROLE_ID`), + CONSTRAINT `ROLE_MAP_FK1` FOREIGN KEY (`ROLE_ID`) REFERENCES `ROLES` (`ROLE_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `SDS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `SDS` ( + `SD_ID` bigint(20) NOT NULL, + `CD_ID` bigint(20) DEFAULT NULL, + `INPUT_FORMAT` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `IS_COMPRESSED` bit(1) NOT NULL, + `IS_STOREDASSUBDIRECTORIES` bit(1) NOT NULL, + `LOCATION` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `NUM_BUCKETS` int(11) NOT NULL, + `OUTPUT_FORMAT` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `SERDE_ID` bigint(20) DEFAULT NULL, + PRIMARY KEY (`SD_ID`), + KEY `SDS_N49` (`SERDE_ID`), + KEY `SDS_N50` (`CD_ID`), + CONSTRAINT `SDS_FK1` FOREIGN KEY (`SERDE_ID`) REFERENCES `SERDES` (`SERDE_ID`), + CONSTRAINT `SDS_FK2` FOREIGN KEY (`CD_ID`) REFERENCES `CDS` (`CD_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `SD_PARAMS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `SD_PARAMS` ( + `SD_ID` bigint(20) NOT NULL, + `PARAM_KEY` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `PARAM_VALUE` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + PRIMARY KEY (`SD_ID`,`PARAM_KEY`), + KEY `SD_PARAMS_N49` (`SD_ID`), + CONSTRAINT `SD_PARAMS_FK1` FOREIGN KEY (`SD_ID`) REFERENCES `SDS` (`SD_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `SEQUENCE_TABLE` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `SEQUENCE_TABLE` ( + `SEQUENCE_NAME` varchar(255) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `NEXT_VAL` bigint(20) NOT NULL, + PRIMARY KEY (`SEQUENCE_NAME`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `SERDES` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `SERDES` ( + `SERDE_ID` bigint(20) NOT NULL, + `NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `SLIB` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + PRIMARY KEY (`SERDE_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `SERDE_PARAMS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `SERDE_PARAMS` ( + `SERDE_ID` bigint(20) NOT NULL, + `PARAM_KEY` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `PARAM_VALUE` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + PRIMARY KEY (`SERDE_ID`,`PARAM_KEY`), + KEY `SERDE_PARAMS_N49` (`SERDE_ID`), + CONSTRAINT `SERDE_PARAMS_FK1` FOREIGN KEY (`SERDE_ID`) REFERENCES `SERDES` (`SERDE_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `SKEWED_COL_NAMES` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `SKEWED_COL_NAMES` ( + `SD_ID` bigint(20) NOT NULL, + `SKEWED_COL_NAME` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `INTEGER_IDX` int(11) NOT NULL, + PRIMARY KEY (`SD_ID`,`INTEGER_IDX`), + KEY `SKEWED_COL_NAMES_N49` (`SD_ID`), + CONSTRAINT `SKEWED_COL_NAMES_FK1` FOREIGN KEY (`SD_ID`) REFERENCES `SDS` (`SD_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `SKEWED_COL_VALUE_LOC_MAP` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `SKEWED_COL_VALUE_LOC_MAP` ( + `SD_ID` bigint(20) NOT NULL, + `STRING_LIST_ID_KID` bigint(20) NOT NULL, + `LOCATION` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + PRIMARY KEY (`SD_ID`,`STRING_LIST_ID_KID`), + KEY `SKEWED_COL_VALUE_LOC_MAP_N49` (`STRING_LIST_ID_KID`), + KEY `SKEWED_COL_VALUE_LOC_MAP_N50` (`SD_ID`), + CONSTRAINT `SKEWED_COL_VALUE_LOC_MAP_FK2` FOREIGN KEY (`STRING_LIST_ID_KID`) REFERENCES `SKEWED_STRING_LIST` (`STRING_LIST_ID`), + CONSTRAINT `SKEWED_COL_VALUE_LOC_MAP_FK1` FOREIGN KEY (`SD_ID`) REFERENCES `SDS` (`SD_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `SKEWED_STRING_LIST` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `SKEWED_STRING_LIST` ( + `STRING_LIST_ID` bigint(20) NOT NULL, + PRIMARY KEY (`STRING_LIST_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `SKEWED_STRING_LIST_VALUES` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `SKEWED_STRING_LIST_VALUES` ( + `STRING_LIST_ID` bigint(20) NOT NULL, + `STRING_LIST_VALUE` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `INTEGER_IDX` int(11) NOT NULL, + PRIMARY KEY (`STRING_LIST_ID`,`INTEGER_IDX`), + KEY `SKEWED_STRING_LIST_VALUES_N49` (`STRING_LIST_ID`), + CONSTRAINT `SKEWED_STRING_LIST_VALUES_FK1` FOREIGN KEY (`STRING_LIST_ID`) REFERENCES `SKEWED_STRING_LIST` (`STRING_LIST_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `SKEWED_VALUES` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `SKEWED_VALUES` ( + `SD_ID_OID` bigint(20) NOT NULL, + `STRING_LIST_ID_EID` bigint(20) NOT NULL, + `INTEGER_IDX` int(11) NOT NULL, + PRIMARY KEY (`SD_ID_OID`,`INTEGER_IDX`), + KEY `SKEWED_VALUES_N50` (`SD_ID_OID`), + KEY `SKEWED_VALUES_N49` (`STRING_LIST_ID_EID`), + CONSTRAINT `SKEWED_VALUES_FK2` FOREIGN KEY (`STRING_LIST_ID_EID`) REFERENCES `SKEWED_STRING_LIST` (`STRING_LIST_ID`), + CONSTRAINT `SKEWED_VALUES_FK1` FOREIGN KEY (`SD_ID_OID`) REFERENCES `SDS` (`SD_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `SORT_COLS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `SORT_COLS` ( + `SD_ID` bigint(20) NOT NULL, + `COLUMN_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `ORDER` int(11) NOT NULL, + `INTEGER_IDX` int(11) NOT NULL, + PRIMARY KEY (`SD_ID`,`INTEGER_IDX`), + KEY `SORT_COLS_N49` (`SD_ID`), + CONSTRAINT `SORT_COLS_FK1` FOREIGN KEY (`SD_ID`) REFERENCES `SDS` (`SD_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `TABLE_PARAMS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `TABLE_PARAMS` ( + `TBL_ID` bigint(20) NOT NULL, + `PARAM_KEY` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `PARAM_VALUE` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + PRIMARY KEY (`TBL_ID`,`PARAM_KEY`), + KEY `TABLE_PARAMS_N49` (`TBL_ID`), + CONSTRAINT `TABLE_PARAMS_FK1` FOREIGN KEY (`TBL_ID`) REFERENCES `TBLS` (`TBL_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `TBLS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `TBLS` ( + `TBL_ID` bigint(20) NOT NULL, + `CREATE_TIME` int(11) NOT NULL, + `DB_ID` bigint(20) DEFAULT NULL, + `LAST_ACCESS_TIME` int(11) NOT NULL, + `OWNER` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `RETENTION` int(11) NOT NULL, + `SD_ID` bigint(20) DEFAULT NULL, + `TBL_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `TBL_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `VIEW_EXPANDED_TEXT` mediumtext, + `VIEW_ORIGINAL_TEXT` mediumtext, + `LINK_TARGET_ID` bigint(20) DEFAULT NULL, + PRIMARY KEY (`TBL_ID`), + UNIQUE KEY `UNIQUETABLE` (`TBL_NAME`,`DB_ID`), + KEY `TBLS_N50` (`SD_ID`), + KEY `TBLS_N49` (`DB_ID`), + KEY `TBLS_N51` (`LINK_TARGET_ID`), + CONSTRAINT `TBLS_FK1` FOREIGN KEY (`SD_ID`) REFERENCES `SDS` (`SD_ID`), + CONSTRAINT `TBLS_FK2` FOREIGN KEY (`DB_ID`) REFERENCES `DBS` (`DB_ID`), + CONSTRAINT `TBLS_FK3` FOREIGN KEY (`LINK_TARGET_ID`) REFERENCES `TBLS` (`TBL_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `TBL_COL_PRIVS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `TBL_COL_PRIVS` ( + `TBL_COLUMN_GRANT_ID` bigint(20) NOT NULL, + `COLUMN_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `CREATE_TIME` int(11) NOT NULL, + `GRANT_OPTION` smallint(6) NOT NULL, + `GRANTOR` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `GRANTOR_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `PRINCIPAL_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `PRINCIPAL_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `TBL_COL_PRIV` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `TBL_ID` bigint(20) DEFAULT NULL, + PRIMARY KEY (`TBL_COLUMN_GRANT_ID`), + KEY `TABLECOLUMNPRIVILEGEINDEX` (`TBL_ID`,`COLUMN_NAME`,`PRINCIPAL_NAME`,`PRINCIPAL_TYPE`,`TBL_COL_PRIV`,`GRANTOR`,`GRANTOR_TYPE`), + KEY `TBL_COL_PRIVS_N49` (`TBL_ID`), + CONSTRAINT `TBL_COL_PRIVS_FK1` FOREIGN KEY (`TBL_ID`) REFERENCES `TBLS` (`TBL_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `TBL_PRIVS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `TBL_PRIVS` ( + `TBL_GRANT_ID` bigint(20) NOT NULL, + `CREATE_TIME` int(11) NOT NULL, + `GRANT_OPTION` smallint(6) NOT NULL, + `GRANTOR` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `GRANTOR_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `PRINCIPAL_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `PRINCIPAL_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `TBL_PRIV` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `TBL_ID` bigint(20) DEFAULT NULL, + PRIMARY KEY (`TBL_GRANT_ID`), + KEY `TBL_PRIVS_N49` (`TBL_ID`), + KEY `TABLEPRIVILEGEINDEX` (`TBL_ID`,`PRINCIPAL_NAME`,`PRINCIPAL_TYPE`,`TBL_PRIV`,`GRANTOR`,`GRANTOR_TYPE`), + CONSTRAINT `TBL_PRIVS_FK1` FOREIGN KEY (`TBL_ID`) REFERENCES `TBLS` (`TBL_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `TAB_COL_STATS` +-- +CREATE TABLE IF NOT EXISTS `TAB_COL_STATS` ( + `CS_ID` bigint(20) NOT NULL, + `DB_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `TABLE_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `COLUMN_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `COLUMN_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `TBL_ID` bigint(20) NOT NULL, + `LONG_LOW_VALUE` bigint(20), + `LONG_HIGH_VALUE` bigint(20), + `DOUBLE_HIGH_VALUE` double(53,4), + `DOUBLE_LOW_VALUE` double(53,4), + `BIG_DECIMAL_LOW_VALUE` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin, + `BIG_DECIMAL_HIGH_VALUE` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin, + `NUM_NULLS` bigint(20) NOT NULL, + `NUM_DISTINCTS` bigint(20), + `AVG_COL_LEN` double(53,4), + `MAX_COL_LEN` bigint(20), + `NUM_TRUES` bigint(20), + `NUM_FALSES` bigint(20), + `LAST_ANALYZED` bigint(20) NOT NULL, + PRIMARY KEY (`CS_ID`), + CONSTRAINT `TAB_COL_STATS_FK` FOREIGN KEY (`TBL_ID`) REFERENCES `TBLS` (`TBL_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +-- +-- Table structure for table `PART_COL_STATS` +-- +CREATE TABLE IF NOT EXISTS `PART_COL_STATS` ( + `CS_ID` bigint(20) NOT NULL, + `DB_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `TABLE_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `PARTITION_NAME` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `COLUMN_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `COLUMN_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `PART_ID` bigint(20) NOT NULL, + `LONG_LOW_VALUE` bigint(20), + `LONG_HIGH_VALUE` bigint(20), + `DOUBLE_HIGH_VALUE` double(53,4), + `DOUBLE_LOW_VALUE` double(53,4), + `BIG_DECIMAL_LOW_VALUE` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin, + `BIG_DECIMAL_HIGH_VALUE` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin, + `NUM_NULLS` bigint(20) NOT NULL, + `NUM_DISTINCTS` bigint(20), + `AVG_COL_LEN` double(53,4), + `MAX_COL_LEN` bigint(20), + `NUM_TRUES` bigint(20), + `NUM_FALSES` bigint(20), + `LAST_ANALYZED` bigint(20) NOT NULL, + PRIMARY KEY (`CS_ID`), + CONSTRAINT `PART_COL_STATS_FK` FOREIGN KEY (`PART_ID`) REFERENCES `PARTITIONS` (`PART_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE INDEX PCS_STATS_IDX ON PART_COL_STATS (DB_NAME,TABLE_NAME,COLUMN_NAME,PARTITION_NAME) USING BTREE; + +-- +-- Table structure for table `TYPES` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `TYPES` ( + `TYPES_ID` bigint(20) NOT NULL, + `TYPE_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `TYPE1` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `TYPE2` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + PRIMARY KEY (`TYPES_ID`), + UNIQUE KEY `UNIQUE_TYPE` (`TYPE_NAME`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `TYPE_FIELDS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `TYPE_FIELDS` ( + `TYPE_NAME` bigint(20) NOT NULL, + `COMMENT` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `FIELD_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `FIELD_TYPE` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `INTEGER_IDX` int(11) NOT NULL, + PRIMARY KEY (`TYPE_NAME`,`FIELD_NAME`), + KEY `TYPE_FIELDS_N49` (`TYPE_NAME`), + CONSTRAINT `TYPE_FIELDS_FK1` FOREIGN KEY (`TYPE_NAME`) REFERENCES `TYPES` (`TYPES_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +-- Table `MASTER_KEYS` for classes [org.apache.hadoop.hive.metastore.model.MMasterKey] +CREATE TABLE IF NOT EXISTS `MASTER_KEYS` +( + `KEY_ID` INTEGER NOT NULL AUTO_INCREMENT, + `MASTER_KEY` VARCHAR(767) BINARY NULL, + PRIMARY KEY (`KEY_ID`) +) ENGINE=INNODB DEFAULT CHARSET=latin1; + +-- Table `DELEGATION_TOKENS` for classes [org.apache.hadoop.hive.metastore.model.MDelegationToken] +CREATE TABLE IF NOT EXISTS `DELEGATION_TOKENS` +( + `TOKEN_IDENT` VARCHAR(767) BINARY NOT NULL, + `TOKEN` VARCHAR(767) BINARY NULL, + PRIMARY KEY (`TOKEN_IDENT`) +) ENGINE=INNODB DEFAULT CHARSET=latin1; + +-- +-- Table structure for VERSION +-- +CREATE TABLE IF NOT EXISTS `VERSION` ( + `VER_ID` BIGINT NOT NULL, + `SCHEMA_VERSION` VARCHAR(127) NOT NULL, + `VERSION_COMMENT` VARCHAR(255), + PRIMARY KEY (`VER_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +-- +-- Table structure for table FUNCS +-- +CREATE TABLE IF NOT EXISTS `FUNCS` ( + `FUNC_ID` BIGINT(20) NOT NULL, + `CLASS_NAME` VARCHAR(4000) CHARACTER SET latin1 COLLATE latin1_bin, + `CREATE_TIME` INT(11) NOT NULL, + `DB_ID` BIGINT(20), + `FUNC_NAME` VARCHAR(128) CHARACTER SET latin1 COLLATE latin1_bin, + `FUNC_TYPE` INT(11) NOT NULL, + `OWNER_NAME` VARCHAR(128) CHARACTER SET latin1 COLLATE latin1_bin, + `OWNER_TYPE` VARCHAR(10) CHARACTER SET latin1 COLLATE latin1_bin, + PRIMARY KEY (`FUNC_ID`), + UNIQUE KEY `UNIQUEFUNCTION` (`FUNC_NAME`, `DB_ID`), + KEY `FUNCS_N49` (`DB_ID`), + CONSTRAINT `FUNCS_FK1` FOREIGN KEY (`DB_ID`) REFERENCES `DBS` (`DB_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +-- +-- Table structure for table FUNC_RU +-- +CREATE TABLE IF NOT EXISTS `FUNC_RU` ( + `FUNC_ID` BIGINT(20) NOT NULL, + `RESOURCE_TYPE` INT(11) NOT NULL, + `RESOURCE_URI` VARCHAR(4000) CHARACTER SET latin1 COLLATE latin1_bin, + `INTEGER_IDX` INT(11) NOT NULL, + PRIMARY KEY (`FUNC_ID`, `INTEGER_IDX`), + CONSTRAINT `FUNC_RU_FK1` FOREIGN KEY (`FUNC_ID`) REFERENCES `FUNCS` (`FUNC_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + + +-- ---------------------------- +-- Transaction and Lock Tables +-- ---------------------------- +SOURCE hive-txn-schema-0.13.0.mysql.sql; + +-- ----------------------------------------------------------------- +-- Record schema version. Should be the last step in the init script +-- ----------------------------------------------------------------- +INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES (1, '0.15.0', 'Hive release version 0.15.0'); + +/*!40101 SET character_set_client = @saved_cs_client */; +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; +/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; + +-- Dump completed on 2012-08-23 0:56:31 diff --git a/metastore/scripts/upgrade/mysql/hive-txn-schema-0.14.0.mysql.sql b/metastore/scripts/upgrade/mysql/hive-txn-schema-0.14.0.mysql.sql new file mode 100644 index 0000000..dfb97f4 --- /dev/null +++ b/metastore/scripts/upgrade/mysql/hive-txn-schema-0.14.0.mysql.sql @@ -0,0 +1,90 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- +-- Tables for transaction management +-- + +CREATE TABLE TXNS ( + TXN_ID bigint PRIMARY KEY, + TXN_STATE char(1) NOT NULL, + TXN_STARTED bigint NOT NULL, + TXN_LAST_HEARTBEAT bigint NOT NULL, + TXN_USER varchar(128) NOT NULL, + TXN_HOST varchar(128) NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE TABLE TXN_COMPONENTS ( + TC_TXNID bigint, + TC_DATABASE varchar(128) NOT NULL, + TC_TABLE varchar(128), + TC_PARTITION varchar(767), + FOREIGN KEY (TC_TXNID) REFERENCES TXNS (TXN_ID) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE TABLE COMPLETED_TXN_COMPONENTS ( + CTC_TXNID bigint, + CTC_DATABASE varchar(128) NOT NULL, + CTC_TABLE varchar(128), + CTC_PARTITION varchar(767) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE TABLE NEXT_TXN_ID ( + NTXN_NEXT bigint NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +INSERT INTO NEXT_TXN_ID VALUES(1); + +CREATE TABLE HIVE_LOCKS ( + HL_LOCK_EXT_ID bigint NOT NULL, + HL_LOCK_INT_ID bigint NOT NULL, + HL_TXNID bigint, + HL_DB varchar(128) NOT NULL, + HL_TABLE varchar(128), + HL_PARTITION varchar(767), + HL_LOCK_STATE char(1) not null, + HL_LOCK_TYPE char(1) not null, + HL_LAST_HEARTBEAT bigint NOT NULL, + HL_ACQUIRED_AT bigint, + HL_USER varchar(128) NOT NULL, + HL_HOST varchar(128) NOT NULL, + PRIMARY KEY(HL_LOCK_EXT_ID, HL_LOCK_INT_ID), + KEY HIVE_LOCK_TXNID_INDEX (HL_TXNID) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE INDEX HL_TXNID_IDX ON HIVE_LOCKS (HL_TXNID); + +CREATE TABLE NEXT_LOCK_ID ( + NL_NEXT bigint NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +INSERT INTO NEXT_LOCK_ID VALUES(1); + +CREATE TABLE COMPACTION_QUEUE ( + CQ_ID bigint PRIMARY KEY, + CQ_DATABASE varchar(128) NOT NULL, + CQ_TABLE varchar(128) NOT NULL, + CQ_PARTITION varchar(767), + CQ_STATE char(1) NOT NULL, + CQ_TYPE char(1) NOT NULL, + CQ_WORKER_ID varchar(128), + CQ_START bigint, + CQ_RUN_AS varchar(128) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE TABLE NEXT_COMPACTION_QUEUE_ID ( + NCQ_NEXT bigint NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1); + + diff --git a/metastore/scripts/upgrade/mysql/upgrade-0.14.0-to-0.15.0.mysql.sql b/metastore/scripts/upgrade/mysql/upgrade-0.14.0-to-0.15.0.mysql.sql new file mode 100644 index 0000000..7484b83 --- /dev/null +++ b/metastore/scripts/upgrade/mysql/upgrade-0.14.0-to-0.15.0.mysql.sql @@ -0,0 +1,5 @@ +SELECT 'Upgrading MetaStore schema from 0.14.0 to 0.15.0' AS ' '; + +UPDATE VERSION SET SCHEMA_VERSION='0.14.0', VERSION_COMMENT='Hive release version 0.15.0' where VER_ID=1; +SELECT 'Finished upgrading MetaStore schema from 0.14.0 to 0.15.0' AS ' '; + diff --git a/metastore/scripts/upgrade/mysql/upgrade.order.mysql b/metastore/scripts/upgrade/mysql/upgrade.order.mysql index 682618a..509e52a 100644 --- a/metastore/scripts/upgrade/mysql/upgrade.order.mysql +++ b/metastore/scripts/upgrade/mysql/upgrade.order.mysql @@ -7,3 +7,4 @@ 0.11.0-to-0.12.0 0.12.0-to-0.13.0 0.13.0-to-0.14.0 +0.14.0-to-0.15.0 diff --git a/metastore/scripts/upgrade/oracle/hive-schema-0.14.0.oracle.sql b/metastore/scripts/upgrade/oracle/hive-schema-0.14.0.oracle.sql index b810b3d..f1f71ce 100644 --- a/metastore/scripts/upgrade/oracle/hive-schema-0.14.0.oracle.sql +++ b/metastore/scripts/upgrade/oracle/hive-schema-0.14.0.oracle.sql @@ -757,8 +757,75 @@ CREATE INDEX FUNC_RU_N49 ON FUNC_RU (FUNC_ID); ------------------------------ -- Transaction and lock tables +-- These are not part of package jdo, so if you are going to regenerate this file you need to manually add the following section back to the file. ------------------------------ -@hive-txn-schema-0.13.0.oracle.sql; +CREATE TABLE TXNS ( + TXN_ID NUMBER(19) PRIMARY KEY, + TXN_STATE char(1) NOT NULL, + TXN_STARTED NUMBER(19) NOT NULL, + TXN_LAST_HEARTBEAT NUMBER(19) NOT NULL, + TXN_USER varchar(128) NOT NULL, + TXN_HOST varchar(128) NOT NULL +); + +CREATE TABLE TXN_COMPONENTS ( + TC_TXNID NUMBER(19) REFERENCES TXNS (TXN_ID), + TC_DATABASE VARCHAR2(128) NOT NULL, + TC_TABLE VARCHAR2(128), + TC_PARTITION VARCHAR2(767) NULL +); + +CREATE TABLE COMPLETED_TXN_COMPONENTS ( + CTC_TXNID NUMBER(19), + CTC_DATABASE varchar(128) NOT NULL, + CTC_TABLE varchar(128), + CTC_PARTITION varchar(767) +); + +CREATE TABLE NEXT_TXN_ID ( + NTXN_NEXT NUMBER(19) NOT NULL +); +INSERT INTO NEXT_TXN_ID VALUES(1); + +CREATE TABLE HIVE_LOCKS ( + HL_LOCK_EXT_ID NUMBER(19) NOT NULL, + HL_LOCK_INT_ID NUMBER(19) NOT NULL, + HL_TXNID NUMBER(19), + HL_DB VARCHAR2(128) NOT NULL, + HL_TABLE VARCHAR2(128), + HL_PARTITION VARCHAR2(767), + HL_LOCK_STATE CHAR(1) NOT NULL, + HL_LOCK_TYPE CHAR(1) NOT NULL, + HL_LAST_HEARTBEAT NUMBER(19) NOT NULL, + HL_ACQUIRED_AT NUMBER(19), + HL_USER varchar(128) NOT NULL, + HL_HOST varchar(128) NOT NULL, + PRIMARY KEY(HL_LOCK_EXT_ID, HL_LOCK_INT_ID) +); + +CREATE INDEX HL_TXNID_INDEX ON HIVE_LOCKS (HL_TXNID); + +CREATE TABLE NEXT_LOCK_ID ( + NL_NEXT NUMBER(19) NOT NULL +); +INSERT INTO NEXT_LOCK_ID VALUES(1); + +CREATE TABLE COMPACTION_QUEUE ( + CQ_ID NUMBER(19) PRIMARY KEY, + CQ_DATABASE varchar(128) NOT NULL, + CQ_TABLE varchar(128) NOT NULL, + CQ_PARTITION varchar(767), + CQ_STATE char(1) NOT NULL, + CQ_TYPE char(1) NOT NULL, + CQ_WORKER_ID varchar(128), + CQ_START NUMBER(19), + CQ_RUN_AS varchar(128) +); + +CREATE TABLE NEXT_COMPACTION_QUEUE_ID ( + NCQ_NEXT NUMBER(19) NOT NULL +); +INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1); -- ----------------------------------------------------------------- -- Record schema version. Should be the last step in the init script diff --git a/metastore/scripts/upgrade/oracle/hive-schema-0.15.0.oracle.sql b/metastore/scripts/upgrade/oracle/hive-schema-0.15.0.oracle.sql new file mode 100644 index 0000000..ec468bb --- /dev/null +++ b/metastore/scripts/upgrade/oracle/hive-schema-0.15.0.oracle.sql @@ -0,0 +1,766 @@ +-- Table SEQUENCE_TABLE is an internal table required by DataNucleus. +-- NOTE: Some versions of SchemaTool do not automatically generate this table. +-- See http://www.datanucleus.org/servlet/jira/browse/NUCRDBMS-416 +CREATE TABLE SEQUENCE_TABLE +( + SEQUENCE_NAME VARCHAR2(255) NOT NULL, + NEXT_VAL NUMBER NOT NULL +); + +ALTER TABLE SEQUENCE_TABLE ADD CONSTRAINT PART_TABLE_PK PRIMARY KEY (SEQUENCE_NAME); + +-- Table NUCLEUS_TABLES is an internal table required by DataNucleus. +-- This table is required if datanucleus.autoStartMechanism=SchemaTable +-- NOTE: Some versions of SchemaTool do not automatically generate this table. +-- See http://www.datanucleus.org/servlet/jira/browse/NUCRDBMS-416 +CREATE TABLE NUCLEUS_TABLES +( + CLASS_NAME VARCHAR2(128) NOT NULL, + TABLE_NAME VARCHAR2(128) NOT NULL, + TYPE VARCHAR2(4) NOT NULL, + OWNER VARCHAR2(2) NOT NULL, + VERSION VARCHAR2(20) NOT NULL, + INTERFACE_NAME VARCHAR2(255) NULL +); + +ALTER TABLE NUCLEUS_TABLES ADD CONSTRAINT NUCLEUS_TABLES_PK PRIMARY KEY (CLASS_NAME); + +-- Table PART_COL_PRIVS for classes [org.apache.hadoop.hive.metastore.model.MPartitionColumnPrivilege] +CREATE TABLE PART_COL_PRIVS +( + PART_COLUMN_GRANT_ID NUMBER NOT NULL, + "COLUMN_NAME" VARCHAR2(128) NULL, + CREATE_TIME NUMBER (10) NOT NULL, + GRANT_OPTION NUMBER (5) NOT NULL, + GRANTOR VARCHAR2(128) NULL, + GRANTOR_TYPE VARCHAR2(128) NULL, + PART_ID NUMBER NULL, + PRINCIPAL_NAME VARCHAR2(128) NULL, + PRINCIPAL_TYPE VARCHAR2(128) NULL, + PART_COL_PRIV VARCHAR2(128) NULL +); + +ALTER TABLE PART_COL_PRIVS ADD CONSTRAINT PART_COL_PRIVS_PK PRIMARY KEY (PART_COLUMN_GRANT_ID); + +-- Table CDS. +CREATE TABLE CDS +( + CD_ID NUMBER NOT NULL +); + +ALTER TABLE CDS ADD CONSTRAINT CDS_PK PRIMARY KEY (CD_ID); + +-- Table COLUMNS_V2 for join relationship +CREATE TABLE COLUMNS_V2 +( + CD_ID NUMBER NOT NULL, + "COMMENT" VARCHAR2(256) NULL, + "COLUMN_NAME" VARCHAR2(128) NOT NULL, + TYPE_NAME VARCHAR2(4000) NOT NULL, + INTEGER_IDX NUMBER(10) NOT NULL +); + +ALTER TABLE COLUMNS_V2 ADD CONSTRAINT COLUMNS_V2_PK PRIMARY KEY (CD_ID,"COLUMN_NAME"); + +-- Table PARTITION_KEY_VALS for join relationship +CREATE TABLE PARTITION_KEY_VALS +( + PART_ID NUMBER NOT NULL, + PART_KEY_VAL VARCHAR2(256) NULL, + INTEGER_IDX NUMBER(10) NOT NULL +); + +ALTER TABLE PARTITION_KEY_VALS ADD CONSTRAINT PARTITION_KEY_VALS_PK PRIMARY KEY (PART_ID,INTEGER_IDX); + +-- Table DBS for classes [org.apache.hadoop.hive.metastore.model.MDatabase] +CREATE TABLE DBS +( + DB_ID NUMBER NOT NULL, + "DESC" VARCHAR2(4000) NULL, + DB_LOCATION_URI VARCHAR2(4000) NOT NULL, + "NAME" VARCHAR2(128) NULL, + OWNER_NAME VARCHAR2(128) NULL, + OWNER_TYPE VARCHAR2(10) NULL +); + +ALTER TABLE DBS ADD CONSTRAINT DBS_PK PRIMARY KEY (DB_ID); + +-- Table PARTITION_PARAMS for join relationship +CREATE TABLE PARTITION_PARAMS +( + PART_ID NUMBER NOT NULL, + PARAM_KEY VARCHAR2(256) NOT NULL, + PARAM_VALUE VARCHAR2(4000) NULL +); + +ALTER TABLE PARTITION_PARAMS ADD CONSTRAINT PARTITION_PARAMS_PK PRIMARY KEY (PART_ID,PARAM_KEY); + +-- Table SERDES for classes [org.apache.hadoop.hive.metastore.model.MSerDeInfo] +CREATE TABLE SERDES +( + SERDE_ID NUMBER NOT NULL, + "NAME" VARCHAR2(128) NULL, + SLIB VARCHAR2(4000) NULL +); + +ALTER TABLE SERDES ADD CONSTRAINT SERDES_PK PRIMARY KEY (SERDE_ID); + +-- Table TYPES for classes [org.apache.hadoop.hive.metastore.model.MType] +CREATE TABLE TYPES +( + TYPES_ID NUMBER NOT NULL, + TYPE_NAME VARCHAR2(128) NULL, + TYPE1 VARCHAR2(767) NULL, + TYPE2 VARCHAR2(767) NULL +); + +ALTER TABLE TYPES ADD CONSTRAINT TYPES_PK PRIMARY KEY (TYPES_ID); + +-- Table PARTITION_KEYS for join relationship +CREATE TABLE PARTITION_KEYS +( + TBL_ID NUMBER NOT NULL, + PKEY_COMMENT VARCHAR2(4000) NULL, + PKEY_NAME VARCHAR2(128) NOT NULL, + PKEY_TYPE VARCHAR2(767) NOT NULL, + INTEGER_IDX NUMBER(10) NOT NULL +); + +ALTER TABLE PARTITION_KEYS ADD CONSTRAINT PARTITION_KEY_PK PRIMARY KEY (TBL_ID,PKEY_NAME); + +-- Table ROLES for classes [org.apache.hadoop.hive.metastore.model.MRole] +CREATE TABLE ROLES +( + ROLE_ID NUMBER NOT NULL, + CREATE_TIME NUMBER (10) NOT NULL, + OWNER_NAME VARCHAR2(128) NULL, + ROLE_NAME VARCHAR2(128) NULL +); + +ALTER TABLE ROLES ADD CONSTRAINT ROLES_PK PRIMARY KEY (ROLE_ID); + +-- Table PARTITIONS for classes [org.apache.hadoop.hive.metastore.model.MPartition] +CREATE TABLE PARTITIONS +( + PART_ID NUMBER NOT NULL, + CREATE_TIME NUMBER (10) NOT NULL, + LAST_ACCESS_TIME NUMBER (10) NOT NULL, + PART_NAME VARCHAR2(767) NULL, + SD_ID NUMBER NULL, + TBL_ID NUMBER NULL +); + +ALTER TABLE PARTITIONS ADD CONSTRAINT PARTITIONS_PK PRIMARY KEY (PART_ID); + +-- Table INDEX_PARAMS for join relationship +CREATE TABLE INDEX_PARAMS +( + INDEX_ID NUMBER NOT NULL, + PARAM_KEY VARCHAR2(256) NOT NULL, + PARAM_VALUE VARCHAR2(4000) NULL +); + +ALTER TABLE INDEX_PARAMS ADD CONSTRAINT INDEX_PARAMS_PK PRIMARY KEY (INDEX_ID,PARAM_KEY); + +-- Table TBL_COL_PRIVS for classes [org.apache.hadoop.hive.metastore.model.MTableColumnPrivilege] +CREATE TABLE TBL_COL_PRIVS +( + TBL_COLUMN_GRANT_ID NUMBER NOT NULL, + "COLUMN_NAME" VARCHAR2(128) NULL, + CREATE_TIME NUMBER (10) NOT NULL, + GRANT_OPTION NUMBER (5) NOT NULL, + GRANTOR VARCHAR2(128) NULL, + GRANTOR_TYPE VARCHAR2(128) NULL, + PRINCIPAL_NAME VARCHAR2(128) NULL, + PRINCIPAL_TYPE VARCHAR2(128) NULL, + TBL_COL_PRIV VARCHAR2(128) NULL, + TBL_ID NUMBER NULL +); + +ALTER TABLE TBL_COL_PRIVS ADD CONSTRAINT TBL_COL_PRIVS_PK PRIMARY KEY (TBL_COLUMN_GRANT_ID); + +-- Table IDXS for classes [org.apache.hadoop.hive.metastore.model.MIndex] +CREATE TABLE IDXS +( + INDEX_ID NUMBER NOT NULL, + CREATE_TIME NUMBER (10) NOT NULL, + DEFERRED_REBUILD NUMBER(1) NOT NULL CHECK (DEFERRED_REBUILD IN (1,0)), + INDEX_HANDLER_CLASS VARCHAR2(4000) NULL, + INDEX_NAME VARCHAR2(128) NULL, + INDEX_TBL_ID NUMBER NULL, + LAST_ACCESS_TIME NUMBER (10) NOT NULL, + ORIG_TBL_ID NUMBER NULL, + SD_ID NUMBER NULL +); + +ALTER TABLE IDXS ADD CONSTRAINT IDXS_PK PRIMARY KEY (INDEX_ID); + +-- Table BUCKETING_COLS for join relationship +CREATE TABLE BUCKETING_COLS +( + SD_ID NUMBER NOT NULL, + BUCKET_COL_NAME VARCHAR2(256) NULL, + INTEGER_IDX NUMBER(10) NOT NULL +); + +ALTER TABLE BUCKETING_COLS ADD CONSTRAINT BUCKETING_COLS_PK PRIMARY KEY (SD_ID,INTEGER_IDX); + +-- Table TYPE_FIELDS for join relationship +CREATE TABLE TYPE_FIELDS +( + TYPE_NAME NUMBER NOT NULL, + "COMMENT" VARCHAR2(256) NULL, + FIELD_NAME VARCHAR2(128) NOT NULL, + FIELD_TYPE VARCHAR2(767) NOT NULL, + INTEGER_IDX NUMBER(10) NOT NULL +); + +ALTER TABLE TYPE_FIELDS ADD CONSTRAINT TYPE_FIELDS_PK PRIMARY KEY (TYPE_NAME,FIELD_NAME); + +-- Table SD_PARAMS for join relationship +CREATE TABLE SD_PARAMS +( + SD_ID NUMBER NOT NULL, + PARAM_KEY VARCHAR2(256) NOT NULL, + PARAM_VALUE VARCHAR2(4000) NULL +); + +ALTER TABLE SD_PARAMS ADD CONSTRAINT SD_PARAMS_PK PRIMARY KEY (SD_ID,PARAM_KEY); + +-- Table GLOBAL_PRIVS for classes [org.apache.hadoop.hive.metastore.model.MGlobalPrivilege] +CREATE TABLE GLOBAL_PRIVS +( + USER_GRANT_ID NUMBER NOT NULL, + CREATE_TIME NUMBER (10) NOT NULL, + GRANT_OPTION NUMBER (5) NOT NULL, + GRANTOR VARCHAR2(128) NULL, + GRANTOR_TYPE VARCHAR2(128) NULL, + PRINCIPAL_NAME VARCHAR2(128) NULL, + PRINCIPAL_TYPE VARCHAR2(128) NULL, + USER_PRIV VARCHAR2(128) NULL +); + +ALTER TABLE GLOBAL_PRIVS ADD CONSTRAINT GLOBAL_PRIVS_PK PRIMARY KEY (USER_GRANT_ID); + +-- Table SDS for classes [org.apache.hadoop.hive.metastore.model.MStorageDescriptor] +CREATE TABLE SDS +( + SD_ID NUMBER NOT NULL, + CD_ID NUMBER NULL, + INPUT_FORMAT VARCHAR2(4000) NULL, + IS_COMPRESSED NUMBER(1) NOT NULL CHECK (IS_COMPRESSED IN (1,0)), + LOCATION VARCHAR2(4000) NULL, + NUM_BUCKETS NUMBER (10) NOT NULL, + OUTPUT_FORMAT VARCHAR2(4000) NULL, + SERDE_ID NUMBER NULL, + IS_STOREDASSUBDIRECTORIES NUMBER(1) NOT NULL CHECK (IS_STOREDASSUBDIRECTORIES IN (1,0)) +); + +ALTER TABLE SDS ADD CONSTRAINT SDS_PK PRIMARY KEY (SD_ID); + +-- Table TABLE_PARAMS for join relationship +CREATE TABLE TABLE_PARAMS +( + TBL_ID NUMBER NOT NULL, + PARAM_KEY VARCHAR2(256) NOT NULL, + PARAM_VALUE VARCHAR2(4000) NULL +); + +ALTER TABLE TABLE_PARAMS ADD CONSTRAINT TABLE_PARAMS_PK PRIMARY KEY (TBL_ID,PARAM_KEY); + +-- Table SORT_COLS for join relationship +CREATE TABLE SORT_COLS +( + SD_ID NUMBER NOT NULL, + "COLUMN_NAME" VARCHAR2(128) NULL, + "ORDER" NUMBER (10) NOT NULL, + INTEGER_IDX NUMBER(10) NOT NULL +); + +ALTER TABLE SORT_COLS ADD CONSTRAINT SORT_COLS_PK PRIMARY KEY (SD_ID,INTEGER_IDX); + +-- Table TBL_PRIVS for classes [org.apache.hadoop.hive.metastore.model.MTablePrivilege] +CREATE TABLE TBL_PRIVS +( + TBL_GRANT_ID NUMBER NOT NULL, + CREATE_TIME NUMBER (10) NOT NULL, + GRANT_OPTION NUMBER (5) NOT NULL, + GRANTOR VARCHAR2(128) NULL, + GRANTOR_TYPE VARCHAR2(128) NULL, + PRINCIPAL_NAME VARCHAR2(128) NULL, + PRINCIPAL_TYPE VARCHAR2(128) NULL, + TBL_PRIV VARCHAR2(128) NULL, + TBL_ID NUMBER NULL +); + +ALTER TABLE TBL_PRIVS ADD CONSTRAINT TBL_PRIVS_PK PRIMARY KEY (TBL_GRANT_ID); + +-- Table DATABASE_PARAMS for join relationship +CREATE TABLE DATABASE_PARAMS +( + DB_ID NUMBER NOT NULL, + PARAM_KEY VARCHAR2(180) NOT NULL, + PARAM_VALUE VARCHAR2(4000) NULL +); + +ALTER TABLE DATABASE_PARAMS ADD CONSTRAINT DATABASE_PARAMS_PK PRIMARY KEY (DB_ID,PARAM_KEY); + +-- Table ROLE_MAP for classes [org.apache.hadoop.hive.metastore.model.MRoleMap] +CREATE TABLE ROLE_MAP +( + ROLE_GRANT_ID NUMBER NOT NULL, + ADD_TIME NUMBER (10) NOT NULL, + GRANT_OPTION NUMBER (5) NOT NULL, + GRANTOR VARCHAR2(128) NULL, + GRANTOR_TYPE VARCHAR2(128) NULL, + PRINCIPAL_NAME VARCHAR2(128) NULL, + PRINCIPAL_TYPE VARCHAR2(128) NULL, + ROLE_ID NUMBER NULL +); + +ALTER TABLE ROLE_MAP ADD CONSTRAINT ROLE_MAP_PK PRIMARY KEY (ROLE_GRANT_ID); + +-- Table SERDE_PARAMS for join relationship +CREATE TABLE SERDE_PARAMS +( + SERDE_ID NUMBER NOT NULL, + PARAM_KEY VARCHAR2(256) NOT NULL, + PARAM_VALUE VARCHAR2(4000) NULL +); + +ALTER TABLE SERDE_PARAMS ADD CONSTRAINT SERDE_PARAMS_PK PRIMARY KEY (SERDE_ID,PARAM_KEY); + +-- Table PART_PRIVS for classes [org.apache.hadoop.hive.metastore.model.MPartitionPrivilege] +CREATE TABLE PART_PRIVS +( + PART_GRANT_ID NUMBER NOT NULL, + CREATE_TIME NUMBER (10) NOT NULL, + GRANT_OPTION NUMBER (5) NOT NULL, + GRANTOR VARCHAR2(128) NULL, + GRANTOR_TYPE VARCHAR2(128) NULL, + PART_ID NUMBER NULL, + PRINCIPAL_NAME VARCHAR2(128) NULL, + PRINCIPAL_TYPE VARCHAR2(128) NULL, + PART_PRIV VARCHAR2(128) NULL +); + +ALTER TABLE PART_PRIVS ADD CONSTRAINT PART_PRIVS_PK PRIMARY KEY (PART_GRANT_ID); + +-- Table DB_PRIVS for classes [org.apache.hadoop.hive.metastore.model.MDBPrivilege] +CREATE TABLE DB_PRIVS +( + DB_GRANT_ID NUMBER NOT NULL, + CREATE_TIME NUMBER (10) NOT NULL, + DB_ID NUMBER NULL, + GRANT_OPTION NUMBER (5) NOT NULL, + GRANTOR VARCHAR2(128) NULL, + GRANTOR_TYPE VARCHAR2(128) NULL, + PRINCIPAL_NAME VARCHAR2(128) NULL, + PRINCIPAL_TYPE VARCHAR2(128) NULL, + DB_PRIV VARCHAR2(128) NULL +); + +ALTER TABLE DB_PRIVS ADD CONSTRAINT DB_PRIVS_PK PRIMARY KEY (DB_GRANT_ID); + +-- Table TBLS for classes [org.apache.hadoop.hive.metastore.model.MTable] +CREATE TABLE TBLS +( + TBL_ID NUMBER NOT NULL, + CREATE_TIME NUMBER (10) NOT NULL, + DB_ID NUMBER NULL, + LAST_ACCESS_TIME NUMBER (10) NOT NULL, + OWNER VARCHAR2(767) NULL, + RETENTION NUMBER (10) NOT NULL, + SD_ID NUMBER NULL, + TBL_NAME VARCHAR2(128) NULL, + TBL_TYPE VARCHAR2(128) NULL, + VIEW_EXPANDED_TEXT CLOB NULL, + VIEW_ORIGINAL_TEXT CLOB NULL +); + +ALTER TABLE TBLS ADD CONSTRAINT TBLS_PK PRIMARY KEY (TBL_ID); + +-- Table PARTITION_EVENTS for classes [org.apache.hadoop.hive.metastore.model.MPartitionEvent] +CREATE TABLE PARTITION_EVENTS +( + PART_NAME_ID NUMBER NOT NULL, + DB_NAME VARCHAR2(128) NULL, + EVENT_TIME NUMBER NOT NULL, + EVENT_TYPE NUMBER (10) NOT NULL, + PARTITION_NAME VARCHAR2(767) NULL, + TBL_NAME VARCHAR2(128) NULL +); + +ALTER TABLE PARTITION_EVENTS ADD CONSTRAINT PARTITION_EVENTS_PK PRIMARY KEY (PART_NAME_ID); + +-- Table SKEWED_STRING_LIST for classes [org.apache.hadoop.hive.metastore.model.MStringList] +CREATE TABLE SKEWED_STRING_LIST +( + STRING_LIST_ID NUMBER NOT NULL +); + +ALTER TABLE SKEWED_STRING_LIST ADD CONSTRAINT SKEWED_STRING_LIST_PK PRIMARY KEY (STRING_LIST_ID); + +CREATE TABLE SKEWED_STRING_LIST_VALUES +( + STRING_LIST_ID NUMBER NOT NULL, + "STRING_LIST_VALUE" VARCHAR2(256) NULL, + INTEGER_IDX NUMBER(10) NOT NULL +); + +ALTER TABLE SKEWED_STRING_LIST_VALUES ADD CONSTRAINT SKEWED_STRING_LIST_VALUES_PK PRIMARY KEY (STRING_LIST_ID,INTEGER_IDX); + +ALTER TABLE SKEWED_STRING_LIST_VALUES ADD CONSTRAINT SKEWED_STRING_LIST_VALUES_FK1 FOREIGN KEY (STRING_LIST_ID) REFERENCES SKEWED_STRING_LIST (STRING_LIST_ID) INITIALLY DEFERRED ; + +CREATE TABLE SKEWED_COL_NAMES +( + SD_ID NUMBER NOT NULL, + "SKEWED_COL_NAME" VARCHAR2(256) NULL, + INTEGER_IDX NUMBER(10) NOT NULL +); + +ALTER TABLE SKEWED_COL_NAMES ADD CONSTRAINT SKEWED_COL_NAMES_PK PRIMARY KEY (SD_ID,INTEGER_IDX); + +ALTER TABLE SKEWED_COL_NAMES ADD CONSTRAINT SKEWED_COL_NAMES_FK1 FOREIGN KEY (SD_ID) REFERENCES SDS (SD_ID) INITIALLY DEFERRED ; + +CREATE TABLE SKEWED_COL_VALUE_LOC_MAP +( + SD_ID NUMBER NOT NULL, + STRING_LIST_ID_KID NUMBER NOT NULL, + "LOCATION" VARCHAR2(4000) NULL +); + +CREATE TABLE MASTER_KEYS +( + KEY_ID NUMBER (10) NOT NULL, + MASTER_KEY VARCHAR2(767) NULL +); + +CREATE TABLE DELEGATION_TOKENS +( + TOKEN_IDENT VARCHAR2(767) NOT NULL, + TOKEN VARCHAR2(767) NULL +); + +ALTER TABLE SKEWED_COL_VALUE_LOC_MAP ADD CONSTRAINT SKEWED_COL_VALUE_LOC_MAP_PK PRIMARY KEY (SD_ID,STRING_LIST_ID_KID); + +ALTER TABLE SKEWED_COL_VALUE_LOC_MAP ADD CONSTRAINT SKEWED_COL_VALUE_LOC_MAP_FK1 FOREIGN KEY (STRING_LIST_ID_KID) REFERENCES SKEWED_STRING_LIST (STRING_LIST_ID) INITIALLY DEFERRED ; + +ALTER TABLE SKEWED_COL_VALUE_LOC_MAP ADD CONSTRAINT SKEWED_COL_VALUE_LOC_MAP_FK2 FOREIGN KEY (SD_ID) REFERENCES SDS (SD_ID) INITIALLY DEFERRED ; + +CREATE TABLE SKEWED_VALUES +( + SD_ID_OID NUMBER NOT NULL, + STRING_LIST_ID_EID NUMBER NOT NULL, + INTEGER_IDX NUMBER(10) NOT NULL +); + +ALTER TABLE SKEWED_VALUES ADD CONSTRAINT SKEWED_VALUES_PK PRIMARY KEY (SD_ID_OID,INTEGER_IDX); + +ALTER TABLE SKEWED_VALUES ADD CONSTRAINT SKEWED_VALUES_FK1 FOREIGN KEY (STRING_LIST_ID_EID) REFERENCES SKEWED_STRING_LIST (STRING_LIST_ID) INITIALLY DEFERRED ; + +ALTER TABLE SKEWED_VALUES ADD CONSTRAINT SKEWED_VALUES_FK2 FOREIGN KEY (SD_ID_OID) REFERENCES SDS (SD_ID) INITIALLY DEFERRED ; + +-- column statistics + +CREATE TABLE TAB_COL_STATS ( + CS_ID NUMBER NOT NULL, + DB_NAME VARCHAR2(128) NOT NULL, + TABLE_NAME VARCHAR2(128) NOT NULL, + COLUMN_NAME VARCHAR2(128) NOT NULL, + COLUMN_TYPE VARCHAR2(128) NOT NULL, + TBL_ID NUMBER NOT NULL, + LONG_LOW_VALUE NUMBER, + LONG_HIGH_VALUE NUMBER, + DOUBLE_LOW_VALUE NUMBER, + DOUBLE_HIGH_VALUE NUMBER, + BIG_DECIMAL_LOW_VALUE VARCHAR2(4000), + BIG_DECIMAL_HIGH_VALUE VARCHAR2(4000), + NUM_NULLS NUMBER NOT NULL, + NUM_DISTINCTS NUMBER, + AVG_COL_LEN NUMBER, + MAX_COL_LEN NUMBER, + NUM_TRUES NUMBER, + NUM_FALSES NUMBER, + LAST_ANALYZED NUMBER NOT NULL +); + +CREATE TABLE VERSION ( + VER_ID NUMBER NOT NULL, + SCHEMA_VERSION VARCHAR(127) NOT NULL, + VERSION_COMMENT VARCHAR(255) +); +ALTER TABLE VERSION ADD CONSTRAINT VERSION_PK PRIMARY KEY (VER_ID); + +ALTER TABLE TAB_COL_STATS ADD CONSTRAINT TAB_COL_STATS_PKEY PRIMARY KEY (CS_ID); + +ALTER TABLE TAB_COL_STATS ADD CONSTRAINT TAB_COL_STATS_FK FOREIGN KEY (TBL_ID) REFERENCES TBLS (TBL_ID) INITIALLY DEFERRED ; + +CREATE INDEX TAB_COL_STATS_N49 ON TAB_COL_STATS(TBL_ID); + +CREATE TABLE PART_COL_STATS ( + CS_ID NUMBER NOT NULL, + DB_NAME VARCHAR2(128) NOT NULL, + TABLE_NAME VARCHAR2(128) NOT NULL, + PARTITION_NAME VARCHAR2(767) NOT NULL, + COLUMN_NAME VARCHAR2(128) NOT NULL, + COLUMN_TYPE VARCHAR2(128) NOT NULL, + PART_ID NUMBER NOT NULL, + LONG_LOW_VALUE NUMBER, + LONG_HIGH_VALUE NUMBER, + DOUBLE_LOW_VALUE NUMBER, + DOUBLE_HIGH_VALUE NUMBER, + BIG_DECIMAL_LOW_VALUE VARCHAR2(4000), + BIG_DECIMAL_HIGH_VALUE VARCHAR2(4000), + NUM_NULLS NUMBER NOT NULL, + NUM_DISTINCTS NUMBER, + AVG_COL_LEN NUMBER, + MAX_COL_LEN NUMBER, + NUM_TRUES NUMBER, + NUM_FALSES NUMBER, + LAST_ANALYZED NUMBER NOT NULL +); + +ALTER TABLE PART_COL_STATS ADD CONSTRAINT PART_COL_STATS_PKEY PRIMARY KEY (CS_ID); + +ALTER TABLE PART_COL_STATS ADD CONSTRAINT PART_COL_STATS_FK FOREIGN KEY (PART_ID) REFERENCES PARTITIONS (PART_ID) INITIALLY DEFERRED; + +CREATE INDEX PART_COL_STATS_N49 ON PART_COL_STATS (PART_ID); + +CREATE INDEX PCS_STATS_IDX ON PART_COL_STATS (DB_NAME,TABLE_NAME,COLUMN_NAME,PARTITION_NAME); + +CREATE TABLE FUNCS ( + FUNC_ID NUMBER NOT NULL, + CLASS_NAME VARCHAR2(4000), + CREATE_TIME NUMBER(10) NOT NULL, + DB_ID NUMBER, + FUNC_NAME VARCHAR2(128), + FUNC_TYPE NUMBER(10) NOT NULL, + OWNER_NAME VARCHAR2(128), + OWNER_TYPE VARCHAR2(10) +); + +ALTER TABLE FUNCS ADD CONSTRAINT FUNCS_PK PRIMARY KEY (FUNC_ID); + +CREATE TABLE FUNC_RU ( + FUNC_ID NUMBER NOT NULL, + RESOURCE_TYPE NUMBER(10) NOT NULL, + RESOURCE_URI VARCHAR2(4000), + INTEGER_IDX NUMBER(10) NOT NULL +); + +ALTER TABLE FUNC_RU ADD CONSTRAINT FUNC_RU_PK PRIMARY KEY (FUNC_ID, INTEGER_IDX); + + +-- Constraints for table PART_COL_PRIVS for class(es) [org.apache.hadoop.hive.metastore.model.MPartitionColumnPrivilege] +ALTER TABLE PART_COL_PRIVS ADD CONSTRAINT PART_COL_PRIVS_FK1 FOREIGN KEY (PART_ID) REFERENCES PARTITIONS (PART_ID) INITIALLY DEFERRED ; + +CREATE INDEX PART_COL_PRIVS_N49 ON PART_COL_PRIVS (PART_ID); + +CREATE INDEX PARTITIONCOLUMNPRIVILEGEINDEX ON PART_COL_PRIVS (PART_ID,"COLUMN_NAME",PRINCIPAL_NAME,PRINCIPAL_TYPE,PART_COL_PRIV,GRANTOR,GRANTOR_TYPE); + + +-- Constraints for table COLUMNS_V2 +ALTER TABLE COLUMNS_V2 ADD CONSTRAINT COLUMNS_V2_FK1 FOREIGN KEY (CD_ID) REFERENCES CDS (CD_ID) INITIALLY DEFERRED ; + +CREATE INDEX COLUMNS_V2_N49 ON COLUMNS_V2 (CD_ID); + + +-- Constraints for table PARTITION_KEY_VALS +ALTER TABLE PARTITION_KEY_VALS ADD CONSTRAINT PARTITION_KEY_VALS_FK1 FOREIGN KEY (PART_ID) REFERENCES PARTITIONS (PART_ID) INITIALLY DEFERRED ; + +CREATE INDEX PARTITION_KEY_VALS_N49 ON PARTITION_KEY_VALS (PART_ID); + + +-- Constraints for table DBS for class(es) [org.apache.hadoop.hive.metastore.model.MDatabase] +CREATE UNIQUE INDEX UNIQUE_DATABASE ON DBS ("NAME"); + + +-- Constraints for table PARTITION_PARAMS +ALTER TABLE PARTITION_PARAMS ADD CONSTRAINT PARTITION_PARAMS_FK1 FOREIGN KEY (PART_ID) REFERENCES PARTITIONS (PART_ID) INITIALLY DEFERRED ; + +CREATE INDEX PARTITION_PARAMS_N49 ON PARTITION_PARAMS (PART_ID); + + +-- Constraints for table SERDES for class(es) [org.apache.hadoop.hive.metastore.model.MSerDeInfo] + +-- Constraints for table TYPES for class(es) [org.apache.hadoop.hive.metastore.model.MType] +CREATE UNIQUE INDEX UNIQUE_TYPE ON TYPES (TYPE_NAME); + + +-- Constraints for table PARTITION_KEYS +ALTER TABLE PARTITION_KEYS ADD CONSTRAINT PARTITION_KEYS_FK1 FOREIGN KEY (TBL_ID) REFERENCES TBLS (TBL_ID) INITIALLY DEFERRED ; + +CREATE INDEX PARTITION_KEYS_N49 ON PARTITION_KEYS (TBL_ID); + + +-- Constraints for table ROLES for class(es) [org.apache.hadoop.hive.metastore.model.MRole] +CREATE UNIQUE INDEX ROLEENTITYINDEX ON ROLES (ROLE_NAME); + + +-- Constraints for table PARTITIONS for class(es) [org.apache.hadoop.hive.metastore.model.MPartition] +ALTER TABLE PARTITIONS ADD CONSTRAINT PARTITIONS_FK1 FOREIGN KEY (TBL_ID) REFERENCES TBLS (TBL_ID) INITIALLY DEFERRED ; + +ALTER TABLE PARTITIONS ADD CONSTRAINT PARTITIONS_FK2 FOREIGN KEY (SD_ID) REFERENCES SDS (SD_ID) INITIALLY DEFERRED ; + +CREATE INDEX PARTITIONS_N49 ON PARTITIONS (SD_ID); + +CREATE INDEX PARTITIONS_N50 ON PARTITIONS (TBL_ID); + +CREATE UNIQUE INDEX UNIQUEPARTITION ON PARTITIONS (PART_NAME,TBL_ID); + + +-- Constraints for table INDEX_PARAMS +ALTER TABLE INDEX_PARAMS ADD CONSTRAINT INDEX_PARAMS_FK1 FOREIGN KEY (INDEX_ID) REFERENCES IDXS (INDEX_ID) INITIALLY DEFERRED ; + +CREATE INDEX INDEX_PARAMS_N49 ON INDEX_PARAMS (INDEX_ID); + + +-- Constraints for table TBL_COL_PRIVS for class(es) [org.apache.hadoop.hive.metastore.model.MTableColumnPrivilege] +ALTER TABLE TBL_COL_PRIVS ADD CONSTRAINT TBL_COL_PRIVS_FK1 FOREIGN KEY (TBL_ID) REFERENCES TBLS (TBL_ID) INITIALLY DEFERRED ; + +CREATE INDEX TABLECOLUMNPRIVILEGEINDEX ON TBL_COL_PRIVS (TBL_ID,"COLUMN_NAME",PRINCIPAL_NAME,PRINCIPAL_TYPE,TBL_COL_PRIV,GRANTOR,GRANTOR_TYPE); + +CREATE INDEX TBL_COL_PRIVS_N49 ON TBL_COL_PRIVS (TBL_ID); + + +-- Constraints for table IDXS for class(es) [org.apache.hadoop.hive.metastore.model.MIndex] +ALTER TABLE IDXS ADD CONSTRAINT IDXS_FK2 FOREIGN KEY (SD_ID) REFERENCES SDS (SD_ID) INITIALLY DEFERRED ; + +ALTER TABLE IDXS ADD CONSTRAINT IDXS_FK1 FOREIGN KEY (ORIG_TBL_ID) REFERENCES TBLS (TBL_ID) INITIALLY DEFERRED ; + +ALTER TABLE IDXS ADD CONSTRAINT IDXS_FK3 FOREIGN KEY (INDEX_TBL_ID) REFERENCES TBLS (TBL_ID) INITIALLY DEFERRED ; + +CREATE UNIQUE INDEX UNIQUEINDEX ON IDXS (INDEX_NAME,ORIG_TBL_ID); + +CREATE INDEX IDXS_N50 ON IDXS (INDEX_TBL_ID); + +CREATE INDEX IDXS_N51 ON IDXS (SD_ID); + +CREATE INDEX IDXS_N49 ON IDXS (ORIG_TBL_ID); + + +-- Constraints for table BUCKETING_COLS +ALTER TABLE BUCKETING_COLS ADD CONSTRAINT BUCKETING_COLS_FK1 FOREIGN KEY (SD_ID) REFERENCES SDS (SD_ID) INITIALLY DEFERRED ; + +CREATE INDEX BUCKETING_COLS_N49 ON BUCKETING_COLS (SD_ID); + + +-- Constraints for table TYPE_FIELDS +ALTER TABLE TYPE_FIELDS ADD CONSTRAINT TYPE_FIELDS_FK1 FOREIGN KEY (TYPE_NAME) REFERENCES TYPES (TYPES_ID) INITIALLY DEFERRED ; + +CREATE INDEX TYPE_FIELDS_N49 ON TYPE_FIELDS (TYPE_NAME); + + +-- Constraints for table SD_PARAMS +ALTER TABLE SD_PARAMS ADD CONSTRAINT SD_PARAMS_FK1 FOREIGN KEY (SD_ID) REFERENCES SDS (SD_ID) INITIALLY DEFERRED ; + +CREATE INDEX SD_PARAMS_N49 ON SD_PARAMS (SD_ID); + + +-- Constraints for table GLOBAL_PRIVS for class(es) [org.apache.hadoop.hive.metastore.model.MGlobalPrivilege] +CREATE UNIQUE INDEX GLOBALPRIVILEGEINDEX ON GLOBAL_PRIVS (PRINCIPAL_NAME,PRINCIPAL_TYPE,USER_PRIV,GRANTOR,GRANTOR_TYPE); + + +-- Constraints for table SDS for class(es) [org.apache.hadoop.hive.metastore.model.MStorageDescriptor] +ALTER TABLE SDS ADD CONSTRAINT SDS_FK1 FOREIGN KEY (SERDE_ID) REFERENCES SERDES (SERDE_ID) INITIALLY DEFERRED ; +ALTER TABLE SDS ADD CONSTRAINT SDS_FK2 FOREIGN KEY (CD_ID) REFERENCES CDS (CD_ID) INITIALLY DEFERRED ; + +CREATE INDEX SDS_N49 ON SDS (SERDE_ID); +CREATE INDEX SDS_N50 ON SDS (CD_ID); + + +-- Constraints for table TABLE_PARAMS +ALTER TABLE TABLE_PARAMS ADD CONSTRAINT TABLE_PARAMS_FK1 FOREIGN KEY (TBL_ID) REFERENCES TBLS (TBL_ID) INITIALLY DEFERRED ; + +CREATE INDEX TABLE_PARAMS_N49 ON TABLE_PARAMS (TBL_ID); + + +-- Constraints for table SORT_COLS +ALTER TABLE SORT_COLS ADD CONSTRAINT SORT_COLS_FK1 FOREIGN KEY (SD_ID) REFERENCES SDS (SD_ID) INITIALLY DEFERRED ; + +CREATE INDEX SORT_COLS_N49 ON SORT_COLS (SD_ID); + + +-- Constraints for table TBL_PRIVS for class(es) [org.apache.hadoop.hive.metastore.model.MTablePrivilege] +ALTER TABLE TBL_PRIVS ADD CONSTRAINT TBL_PRIVS_FK1 FOREIGN KEY (TBL_ID) REFERENCES TBLS (TBL_ID) INITIALLY DEFERRED ; + +CREATE INDEX TBL_PRIVS_N49 ON TBL_PRIVS (TBL_ID); + +CREATE INDEX TABLEPRIVILEGEINDEX ON TBL_PRIVS (TBL_ID,PRINCIPAL_NAME,PRINCIPAL_TYPE,TBL_PRIV,GRANTOR,GRANTOR_TYPE); + + +-- Constraints for table DATABASE_PARAMS +ALTER TABLE DATABASE_PARAMS ADD CONSTRAINT DATABASE_PARAMS_FK1 FOREIGN KEY (DB_ID) REFERENCES DBS (DB_ID) INITIALLY DEFERRED ; + +CREATE INDEX DATABASE_PARAMS_N49 ON DATABASE_PARAMS (DB_ID); + + +-- Constraints for table ROLE_MAP for class(es) [org.apache.hadoop.hive.metastore.model.MRoleMap] +ALTER TABLE ROLE_MAP ADD CONSTRAINT ROLE_MAP_FK1 FOREIGN KEY (ROLE_ID) REFERENCES ROLES (ROLE_ID) INITIALLY DEFERRED ; + +CREATE INDEX ROLE_MAP_N49 ON ROLE_MAP (ROLE_ID); + +CREATE UNIQUE INDEX USERROLEMAPINDEX ON ROLE_MAP (PRINCIPAL_NAME,ROLE_ID,GRANTOR,GRANTOR_TYPE); + + +-- Constraints for table SERDE_PARAMS +ALTER TABLE SERDE_PARAMS ADD CONSTRAINT SERDE_PARAMS_FK1 FOREIGN KEY (SERDE_ID) REFERENCES SERDES (SERDE_ID) INITIALLY DEFERRED ; + +CREATE INDEX SERDE_PARAMS_N49 ON SERDE_PARAMS (SERDE_ID); + + +-- Constraints for table PART_PRIVS for class(es) [org.apache.hadoop.hive.metastore.model.MPartitionPrivilege] +ALTER TABLE PART_PRIVS ADD CONSTRAINT PART_PRIVS_FK1 FOREIGN KEY (PART_ID) REFERENCES PARTITIONS (PART_ID) INITIALLY DEFERRED ; + +CREATE INDEX PARTPRIVILEGEINDEX ON PART_PRIVS (PART_ID,PRINCIPAL_NAME,PRINCIPAL_TYPE,PART_PRIV,GRANTOR,GRANTOR_TYPE); + +CREATE INDEX PART_PRIVS_N49 ON PART_PRIVS (PART_ID); + + +-- Constraints for table DB_PRIVS for class(es) [org.apache.hadoop.hive.metastore.model.MDBPrivilege] +ALTER TABLE DB_PRIVS ADD CONSTRAINT DB_PRIVS_FK1 FOREIGN KEY (DB_ID) REFERENCES DBS (DB_ID) INITIALLY DEFERRED ; + +CREATE UNIQUE INDEX DBPRIVILEGEINDEX ON DB_PRIVS (DB_ID,PRINCIPAL_NAME,PRINCIPAL_TYPE,DB_PRIV,GRANTOR,GRANTOR_TYPE); + +CREATE INDEX DB_PRIVS_N49 ON DB_PRIVS (DB_ID); + + +-- Constraints for table TBLS for class(es) [org.apache.hadoop.hive.metastore.model.MTable] +ALTER TABLE TBLS ADD CONSTRAINT TBLS_FK2 FOREIGN KEY (DB_ID) REFERENCES DBS (DB_ID) INITIALLY DEFERRED ; + +ALTER TABLE TBLS ADD CONSTRAINT TBLS_FK1 FOREIGN KEY (SD_ID) REFERENCES SDS (SD_ID) INITIALLY DEFERRED ; + +CREATE INDEX TBLS_N49 ON TBLS (DB_ID); + +CREATE UNIQUE INDEX UNIQUETABLE ON TBLS (TBL_NAME,DB_ID); + +CREATE INDEX TBLS_N50 ON TBLS (SD_ID); + + +-- Constraints for table PARTITION_EVENTS for class(es) [org.apache.hadoop.hive.metastore.model.MPartitionEvent] +CREATE INDEX PARTITIONEVENTINDEX ON PARTITION_EVENTS (PARTITION_NAME); + + +-- Constraints for table FUNCS for class(es) [org.apache.hadoop.hive.metastore.model.MFunctions] +ALTER TABLE FUNCS ADD CONSTRAINT FUNCS_FK1 FOREIGN KEY (DB_ID) REFERENCES DBS (DB_ID) INITIALLY DEFERRED; + +CREATE UNIQUE INDEX UNIQUEFUNCTION ON FUNCS (FUNC_NAME, DB_ID); + +CREATE INDEX FUNCS_N49 ON FUNCS (DB_ID); + + +-- Constraints for table FUNC_RU for class(es) [org.apache.hadoop.hive.metastore.model.MFunctions] +ALTER TABLE FUNC_RU ADD CONSTRAINT FUNC_RU_FK1 FOREIGN KEY (FUNC_ID) REFERENCES FUNCS (FUNC_ID) INITIALLY DEFERRED; + +CREATE INDEX FUNC_RU_N49 ON FUNC_RU (FUNC_ID); + +------------------------------ +-- Transaction and lock tables +------------------------------ +@hive-txn-schema-0.13.0.oracle.sql; + +-- ----------------------------------------------------------------- +-- Record schema version. Should be the last step in the init script +-- ----------------------------------------------------------------- +INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES (1, '0.15.0', 'Hive release version 0.15.0'); diff --git a/metastore/scripts/upgrade/oracle/hive-txn-schema-0.14.0.oracle.sql b/metastore/scripts/upgrade/oracle/hive-txn-schema-0.14.0.oracle.sql new file mode 100644 index 0000000..7435ea8 --- /dev/null +++ b/metastore/scripts/upgrade/oracle/hive-txn-schema-0.14.0.oracle.sql @@ -0,0 +1,88 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the License); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an AS IS BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- +-- Tables for transaction management +-- + +CREATE TABLE TXNS ( + TXN_ID NUMBER(19) PRIMARY KEY, + TXN_STATE char(1) NOT NULL, + TXN_STARTED NUMBER(19) NOT NULL, + TXN_LAST_HEARTBEAT NUMBER(19) NOT NULL, + TXN_USER varchar(128) NOT NULL, + TXN_HOST varchar(128) NOT NULL +); + +CREATE TABLE TXN_COMPONENTS ( + TC_TXNID NUMBER(19) REFERENCES TXNS (TXN_ID), + TC_DATABASE VARCHAR2(128) NOT NULL, + TC_TABLE VARCHAR2(128), + TC_PARTITION VARCHAR2(767) NULL +); + +CREATE TABLE COMPLETED_TXN_COMPONENTS ( + CTC_TXNID NUMBER(19), + CTC_DATABASE varchar(128) NOT NULL, + CTC_TABLE varchar(128), + CTC_PARTITION varchar(767) +); + +CREATE TABLE NEXT_TXN_ID ( + NTXN_NEXT NUMBER(19) NOT NULL +); +INSERT INTO NEXT_TXN_ID VALUES(1); + +CREATE TABLE HIVE_LOCKS ( + HL_LOCK_EXT_ID NUMBER(19) NOT NULL, + HL_LOCK_INT_ID NUMBER(19) NOT NULL, + HL_TXNID NUMBER(19), + HL_DB VARCHAR2(128) NOT NULL, + HL_TABLE VARCHAR2(128), + HL_PARTITION VARCHAR2(767), + HL_LOCK_STATE CHAR(1) NOT NULL, + HL_LOCK_TYPE CHAR(1) NOT NULL, + HL_LAST_HEARTBEAT NUMBER(19) NOT NULL, + HL_ACQUIRED_AT NUMBER(19), + HL_USER varchar(128) NOT NULL, + HL_HOST varchar(128) NOT NULL, + PRIMARY KEY(HL_LOCK_EXT_ID, HL_LOCK_INT_ID) +); + +CREATE INDEX HL_TXNID_INDEX ON HIVE_LOCKS (HL_TXNID); + +CREATE TABLE NEXT_LOCK_ID ( + NL_NEXT NUMBER(19) NOT NULL +); +INSERT INTO NEXT_LOCK_ID VALUES(1); + +CREATE TABLE COMPACTION_QUEUE ( + CQ_ID NUMBER(19) PRIMARY KEY, + CQ_DATABASE varchar(128) NOT NULL, + CQ_TABLE varchar(128) NOT NULL, + CQ_PARTITION varchar(767), + CQ_STATE char(1) NOT NULL, + CQ_TYPE char(1) NOT NULL, + CQ_WORKER_ID varchar(128), + CQ_START NUMBER(19), + CQ_RUN_AS varchar(128) +); + +CREATE TABLE NEXT_COMPACTION_QUEUE_ID ( + NCQ_NEXT NUMBER(19) NOT NULL +); +INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1); + + diff --git a/metastore/scripts/upgrade/oracle/upgrade-0.14.0-to-0.15.0.oracle.sql b/metastore/scripts/upgrade/oracle/upgrade-0.14.0-to-0.15.0.oracle.sql new file mode 100644 index 0000000..54b0766 --- /dev/null +++ b/metastore/scripts/upgrade/oracle/upgrade-0.14.0-to-0.15.0.oracle.sql @@ -0,0 +1,4 @@ +SELECT 'Upgrading MetaStore schema from 0.14.0 to 0.15.0' AS Status from dual; + +UPDATE VERSION SET SCHEMA_VERSION='0.15.0', VERSION_COMMENT='Hive release version 0.15.0' where VER_ID=1; +SELECT 'Finished upgrading MetaStore schema from 0.14.0 to 0.15.0' AS Status from dual; diff --git a/metastore/scripts/upgrade/oracle/upgrade.order.oracle b/metastore/scripts/upgrade/oracle/upgrade.order.oracle index 503d598..36e168a 100644 --- a/metastore/scripts/upgrade/oracle/upgrade.order.oracle +++ b/metastore/scripts/upgrade/oracle/upgrade.order.oracle @@ -3,3 +3,4 @@ 0.11.0-to-0.12.0 0.12.0-to-0.13.0 0.13.0-to-0.14.0 +0.14.0-to-0.15.0 diff --git a/metastore/scripts/upgrade/postgres/hive-schema-0.14.0.postgres.sql b/metastore/scripts/upgrade/postgres/hive-schema-0.14.0.postgres.sql index 5358f50..22347c8 100644 --- a/metastore/scripts/upgrade/postgres/hive-schema-0.14.0.postgres.sql +++ b/metastore/scripts/upgrade/postgres/hive-schema-0.14.0.postgres.sql @@ -1465,8 +1465,75 @@ GRANT ALL ON SCHEMA public TO PUBLIC; ------------------------------ -- Transaction and lock tables +-- These are not part of package jdo, so if you are going to regenerate this file you need to manually add the following section back to the file. ------------------------------ -\i hive-txn-schema-0.13.0.postgres.sql; +CREATE TABLE "txns" ( + "txn_id" bigint PRIMARY KEY, + "txn_state" char(1) NOT NULL, + "txn_started" bigint NOT NULL, + "txn_last_heartbeat" bigint NOT NULL, + "txn_user" varchar(128) NOT NULL, + "txn_host" varchar(128) NOT NULL +); + +CREATE TABLE "txn_components" ( + "tc_txnid" bigint REFERENCES "txns" ("txn_id"), + "tc_database" varchar(128) NOT NULL, + "tc_table" varchar(128), + "tc_partition" varchar(767) DEFAULT NULL +); + +CREATE TABLE "completed_txn_components" ( + "ctc_txnid" bigint, + "ctc_database" varchar(128) NOT NULL, + "ctc_table" varchar(128), + "ctc_partition" varchar(767) +); + +CREATE TABLE "next_txn_id" ( + "ntxn_next" bigint NOT NULL +); +INSERT INTO "next_txn_id" VALUES(1); + +CREATE TABLE "hive_locks" ( + "hl_lock_ext_id" bigint NOT NULL, + "hl_lock_int_id" bigint NOT NULL, + "hl_txnid" bigint, + "hl_db" varchar(128) NOT NULL, + "hl_table" varchar(128), + "hl_partition" varchar(767) DEFAULT NULL, + "hl_lock_state" char(1) NOT NULL, + "hl_lock_type" char(1) NOT NULL, + "hl_last_heartbeat" bigint NOT NULL, + "hl_acquired_at" bigint, + "hl_user" varchar(128) NOT NULL, + "hl_host" varchar(128) NOT NULL, + PRIMARY KEY("hl_lock_ext_id", "hl_lock_int_id") +); + +CREATE INDEX "hl_txnid_index" ON "hive_locks" USING hash ("hl_txnid"); + +CREATE TABLE "next_lock_id" ( + "nl_next" bigint NOT NULL +); +INSERT INTO "next_lock_id" VALUES(1); + +CREATE TABLE "compaction_queue" ( + "cq_id" bigint PRIMARY KEY, + "cq_database" varchar(128) NOT NULL, + "cq_table" varchar(128) NOT NULL, + "cq_partition" varchar(767), + "cq_state" char(1) NOT NULL, + "cq_type" char(1) NOT NULL, + "cq_worker_id" varchar(128), + "cq_start" bigint, + "cq_run_as" varchar(128) +); + +CREATE TABLE "next_compaction_queue_id" ( + "ncq_next" bigint NOT NULL +); +INSERT INTO "next_compaction_queue_id" VALUES(1); -- ----------------------------------------------------------------- -- Record schema version. Should be the last step in the init script diff --git a/metastore/scripts/upgrade/postgres/hive-schema-0.15.0.postgres.sql b/metastore/scripts/upgrade/postgres/hive-schema-0.15.0.postgres.sql new file mode 100644 index 0000000..dd4fe12 --- /dev/null +++ b/metastore/scripts/upgrade/postgres/hive-schema-0.15.0.postgres.sql @@ -0,0 +1,1474 @@ +-- +-- PostgreSQL database dump +-- + +SET statement_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = off; +SET check_function_bodies = false; +SET client_min_messages = warning; +SET escape_string_warning = off; + +SET search_path = public, pg_catalog; + +SET default_tablespace = ''; + +SET default_with_oids = false; + +-- +-- Name: BUCKETING_COLS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "BUCKETING_COLS" ( + "SD_ID" bigint NOT NULL, + "BUCKET_COL_NAME" character varying(256) DEFAULT NULL::character varying, + "INTEGER_IDX" bigint NOT NULL +); + + +-- +-- Name: CDS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "CDS" ( + "CD_ID" bigint NOT NULL +); + + +-- +-- Name: COLUMNS_OLD; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "COLUMNS_OLD" ( + "SD_ID" bigint NOT NULL, + "COMMENT" character varying(256) DEFAULT NULL::character varying, + "COLUMN_NAME" character varying(128) NOT NULL, + "TYPE_NAME" character varying(4000) NOT NULL, + "INTEGER_IDX" bigint NOT NULL +); + + +-- +-- Name: COLUMNS_V2; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "COLUMNS_V2" ( + "CD_ID" bigint NOT NULL, + "COMMENT" character varying(4000), + "COLUMN_NAME" character varying(128) NOT NULL, + "TYPE_NAME" character varying(4000), + "INTEGER_IDX" integer NOT NULL +); + + +-- +-- Name: DATABASE_PARAMS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "DATABASE_PARAMS" ( + "DB_ID" bigint NOT NULL, + "PARAM_KEY" character varying(180) NOT NULL, + "PARAM_VALUE" character varying(4000) DEFAULT NULL::character varying +); + + +-- +-- Name: DBS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "DBS" ( + "DB_ID" bigint NOT NULL, + "DESC" character varying(4000) DEFAULT NULL::character varying, + "DB_LOCATION_URI" character varying(4000) NOT NULL, + "NAME" character varying(128) DEFAULT NULL::character varying, + "OWNER_NAME" character varying(128) DEFAULT NULL::character varying, + "OWNER_TYPE" character varying(10) DEFAULT NULL::character varying +); + + +-- +-- Name: DB_PRIVS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "DB_PRIVS" ( + "DB_GRANT_ID" bigint NOT NULL, + "CREATE_TIME" bigint NOT NULL, + "DB_ID" bigint, + "GRANT_OPTION" smallint NOT NULL, + "GRANTOR" character varying(128) DEFAULT NULL::character varying, + "GRANTOR_TYPE" character varying(128) DEFAULT NULL::character varying, + "PRINCIPAL_NAME" character varying(128) DEFAULT NULL::character varying, + "PRINCIPAL_TYPE" character varying(128) DEFAULT NULL::character varying, + "DB_PRIV" character varying(128) DEFAULT NULL::character varying +); + + +-- +-- Name: GLOBAL_PRIVS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "GLOBAL_PRIVS" ( + "USER_GRANT_ID" bigint NOT NULL, + "CREATE_TIME" bigint NOT NULL, + "GRANT_OPTION" smallint NOT NULL, + "GRANTOR" character varying(128) DEFAULT NULL::character varying, + "GRANTOR_TYPE" character varying(128) DEFAULT NULL::character varying, + "PRINCIPAL_NAME" character varying(128) DEFAULT NULL::character varying, + "PRINCIPAL_TYPE" character varying(128) DEFAULT NULL::character varying, + "USER_PRIV" character varying(128) DEFAULT NULL::character varying +); + + +-- +-- Name: IDXS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "IDXS" ( + "INDEX_ID" bigint NOT NULL, + "CREATE_TIME" bigint NOT NULL, + "DEFERRED_REBUILD" boolean NOT NULL, + "INDEX_HANDLER_CLASS" character varying(4000) DEFAULT NULL::character varying, + "INDEX_NAME" character varying(128) DEFAULT NULL::character varying, + "INDEX_TBL_ID" bigint, + "LAST_ACCESS_TIME" bigint NOT NULL, + "ORIG_TBL_ID" bigint, + "SD_ID" bigint +); + + +-- +-- Name: INDEX_PARAMS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "INDEX_PARAMS" ( + "INDEX_ID" bigint NOT NULL, + "PARAM_KEY" character varying(256) NOT NULL, + "PARAM_VALUE" character varying(4000) DEFAULT NULL::character varying +); + + +-- +-- Name: NUCLEUS_TABLES; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "NUCLEUS_TABLES" ( + "CLASS_NAME" character varying(128) NOT NULL, + "TABLE_NAME" character varying(128) NOT NULL, + "TYPE" character varying(4) NOT NULL, + "OWNER" character varying(2) NOT NULL, + "VERSION" character varying(20) NOT NULL, + "INTERFACE_NAME" character varying(255) DEFAULT NULL::character varying +); + + +-- +-- Name: PARTITIONS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "PARTITIONS" ( + "PART_ID" bigint NOT NULL, + "CREATE_TIME" bigint NOT NULL, + "LAST_ACCESS_TIME" bigint NOT NULL, + "PART_NAME" character varying(767) DEFAULT NULL::character varying, + "SD_ID" bigint, + "TBL_ID" bigint +); + + +-- +-- Name: PARTITION_EVENTS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "PARTITION_EVENTS" ( + "PART_NAME_ID" bigint NOT NULL, + "DB_NAME" character varying(128), + "EVENT_TIME" bigint NOT NULL, + "EVENT_TYPE" integer NOT NULL, + "PARTITION_NAME" character varying(767), + "TBL_NAME" character varying(128) +); + + +-- +-- Name: PARTITION_KEYS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "PARTITION_KEYS" ( + "TBL_ID" bigint NOT NULL, + "PKEY_COMMENT" character varying(4000) DEFAULT NULL::character varying, + "PKEY_NAME" character varying(128) NOT NULL, + "PKEY_TYPE" character varying(767) NOT NULL, + "INTEGER_IDX" bigint NOT NULL +); + + +-- +-- Name: PARTITION_KEY_VALS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "PARTITION_KEY_VALS" ( + "PART_ID" bigint NOT NULL, + "PART_KEY_VAL" character varying(256) DEFAULT NULL::character varying, + "INTEGER_IDX" bigint NOT NULL +); + + +-- +-- Name: PARTITION_PARAMS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "PARTITION_PARAMS" ( + "PART_ID" bigint NOT NULL, + "PARAM_KEY" character varying(256) NOT NULL, + "PARAM_VALUE" character varying(4000) DEFAULT NULL::character varying +); + + +-- +-- Name: PART_COL_PRIVS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "PART_COL_PRIVS" ( + "PART_COLUMN_GRANT_ID" bigint NOT NULL, + "COLUMN_NAME" character varying(128) DEFAULT NULL::character varying, + "CREATE_TIME" bigint NOT NULL, + "GRANT_OPTION" smallint NOT NULL, + "GRANTOR" character varying(128) DEFAULT NULL::character varying, + "GRANTOR_TYPE" character varying(128) DEFAULT NULL::character varying, + "PART_ID" bigint, + "PRINCIPAL_NAME" character varying(128) DEFAULT NULL::character varying, + "PRINCIPAL_TYPE" character varying(128) DEFAULT NULL::character varying, + "PART_COL_PRIV" character varying(128) DEFAULT NULL::character varying +); + + +-- +-- Name: PART_PRIVS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "PART_PRIVS" ( + "PART_GRANT_ID" bigint NOT NULL, + "CREATE_TIME" bigint NOT NULL, + "GRANT_OPTION" smallint NOT NULL, + "GRANTOR" character varying(128) DEFAULT NULL::character varying, + "GRANTOR_TYPE" character varying(128) DEFAULT NULL::character varying, + "PART_ID" bigint, + "PRINCIPAL_NAME" character varying(128) DEFAULT NULL::character varying, + "PRINCIPAL_TYPE" character varying(128) DEFAULT NULL::character varying, + "PART_PRIV" character varying(128) DEFAULT NULL::character varying +); + + +-- +-- Name: ROLES; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "ROLES" ( + "ROLE_ID" bigint NOT NULL, + "CREATE_TIME" bigint NOT NULL, + "OWNER_NAME" character varying(128) DEFAULT NULL::character varying, + "ROLE_NAME" character varying(128) DEFAULT NULL::character varying +); + + +-- +-- Name: ROLE_MAP; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "ROLE_MAP" ( + "ROLE_GRANT_ID" bigint NOT NULL, + "ADD_TIME" bigint NOT NULL, + "GRANT_OPTION" smallint NOT NULL, + "GRANTOR" character varying(128) DEFAULT NULL::character varying, + "GRANTOR_TYPE" character varying(128) DEFAULT NULL::character varying, + "PRINCIPAL_NAME" character varying(128) DEFAULT NULL::character varying, + "PRINCIPAL_TYPE" character varying(128) DEFAULT NULL::character varying, + "ROLE_ID" bigint +); + + +-- +-- Name: SDS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "SDS" ( + "SD_ID" bigint NOT NULL, + "INPUT_FORMAT" character varying(4000) DEFAULT NULL::character varying, + "IS_COMPRESSED" boolean NOT NULL, + "LOCATION" character varying(4000) DEFAULT NULL::character varying, + "NUM_BUCKETS" bigint NOT NULL, + "OUTPUT_FORMAT" character varying(4000) DEFAULT NULL::character varying, + "SERDE_ID" bigint, + "CD_ID" bigint, + "IS_STOREDASSUBDIRECTORIES" boolean NOT NULL +); + + +-- +-- Name: SD_PARAMS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "SD_PARAMS" ( + "SD_ID" bigint NOT NULL, + "PARAM_KEY" character varying(256) NOT NULL, + "PARAM_VALUE" character varying(4000) DEFAULT NULL::character varying +); + + +-- +-- Name: SEQUENCE_TABLE; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "SEQUENCE_TABLE" ( + "SEQUENCE_NAME" character varying(255) NOT NULL, + "NEXT_VAL" bigint NOT NULL +); + + +-- +-- Name: SERDES; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "SERDES" ( + "SERDE_ID" bigint NOT NULL, + "NAME" character varying(128) DEFAULT NULL::character varying, + "SLIB" character varying(4000) DEFAULT NULL::character varying +); + + +-- +-- Name: SERDE_PARAMS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "SERDE_PARAMS" ( + "SERDE_ID" bigint NOT NULL, + "PARAM_KEY" character varying(256) NOT NULL, + "PARAM_VALUE" character varying(4000) DEFAULT NULL::character varying +); + + +-- +-- Name: SORT_COLS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "SORT_COLS" ( + "SD_ID" bigint NOT NULL, + "COLUMN_NAME" character varying(128) DEFAULT NULL::character varying, + "ORDER" bigint NOT NULL, + "INTEGER_IDX" bigint NOT NULL +); + + +-- +-- Name: TABLE_PARAMS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "TABLE_PARAMS" ( + "TBL_ID" bigint NOT NULL, + "PARAM_KEY" character varying(256) NOT NULL, + "PARAM_VALUE" character varying(4000) DEFAULT NULL::character varying +); + + +-- +-- Name: TBLS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "TBLS" ( + "TBL_ID" bigint NOT NULL, + "CREATE_TIME" bigint NOT NULL, + "DB_ID" bigint, + "LAST_ACCESS_TIME" bigint NOT NULL, + "OWNER" character varying(767) DEFAULT NULL::character varying, + "RETENTION" bigint NOT NULL, + "SD_ID" bigint, + "TBL_NAME" character varying(128) DEFAULT NULL::character varying, + "TBL_TYPE" character varying(128) DEFAULT NULL::character varying, + "VIEW_EXPANDED_TEXT" text, + "VIEW_ORIGINAL_TEXT" text +); + + +-- +-- Name: TBL_COL_PRIVS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "TBL_COL_PRIVS" ( + "TBL_COLUMN_GRANT_ID" bigint NOT NULL, + "COLUMN_NAME" character varying(128) DEFAULT NULL::character varying, + "CREATE_TIME" bigint NOT NULL, + "GRANT_OPTION" smallint NOT NULL, + "GRANTOR" character varying(128) DEFAULT NULL::character varying, + "GRANTOR_TYPE" character varying(128) DEFAULT NULL::character varying, + "PRINCIPAL_NAME" character varying(128) DEFAULT NULL::character varying, + "PRINCIPAL_TYPE" character varying(128) DEFAULT NULL::character varying, + "TBL_COL_PRIV" character varying(128) DEFAULT NULL::character varying, + "TBL_ID" bigint +); + + +-- +-- Name: TBL_PRIVS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "TBL_PRIVS" ( + "TBL_GRANT_ID" bigint NOT NULL, + "CREATE_TIME" bigint NOT NULL, + "GRANT_OPTION" smallint NOT NULL, + "GRANTOR" character varying(128) DEFAULT NULL::character varying, + "GRANTOR_TYPE" character varying(128) DEFAULT NULL::character varying, + "PRINCIPAL_NAME" character varying(128) DEFAULT NULL::character varying, + "PRINCIPAL_TYPE" character varying(128) DEFAULT NULL::character varying, + "TBL_PRIV" character varying(128) DEFAULT NULL::character varying, + "TBL_ID" bigint +); + + +-- +-- Name: TYPES; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "TYPES" ( + "TYPES_ID" bigint NOT NULL, + "TYPE_NAME" character varying(128) DEFAULT NULL::character varying, + "TYPE1" character varying(767) DEFAULT NULL::character varying, + "TYPE2" character varying(767) DEFAULT NULL::character varying +); + + +-- +-- Name: TYPE_FIELDS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "TYPE_FIELDS" ( + "TYPE_NAME" bigint NOT NULL, + "COMMENT" character varying(256) DEFAULT NULL::character varying, + "FIELD_NAME" character varying(128) NOT NULL, + "FIELD_TYPE" character varying(767) NOT NULL, + "INTEGER_IDX" bigint NOT NULL +); + +CREATE TABLE "SKEWED_STRING_LIST" ( + "STRING_LIST_ID" bigint NOT NULL +); + +CREATE TABLE "SKEWED_STRING_LIST_VALUES" ( + "STRING_LIST_ID" bigint NOT NULL, + "STRING_LIST_VALUE" character varying(256) DEFAULT NULL::character varying, + "INTEGER_IDX" bigint NOT NULL +); + +CREATE TABLE "SKEWED_COL_NAMES" ( + "SD_ID" bigint NOT NULL, + "SKEWED_COL_NAME" character varying(256) DEFAULT NULL::character varying, + "INTEGER_IDX" bigint NOT NULL +); + +CREATE TABLE "SKEWED_COL_VALUE_LOC_MAP" ( + "SD_ID" bigint NOT NULL, + "STRING_LIST_ID_KID" bigint NOT NULL, + "LOCATION" character varying(4000) DEFAULT NULL::character varying +); + +CREATE TABLE "SKEWED_VALUES" ( + "SD_ID_OID" bigint NOT NULL, + "STRING_LIST_ID_EID" bigint NOT NULL, + "INTEGER_IDX" bigint NOT NULL +); + + +-- +-- Name: TAB_COL_STATS Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "MASTER_KEYS" +( + "KEY_ID" SERIAL, + "MASTER_KEY" varchar(767) NULL, + PRIMARY KEY ("KEY_ID") +); + +CREATE TABLE "DELEGATION_TOKENS" +( + "TOKEN_IDENT" varchar(767) NOT NULL, + "TOKEN" varchar(767) NULL, + PRIMARY KEY ("TOKEN_IDENT") +); + +CREATE TABLE "TAB_COL_STATS" ( + "CS_ID" bigint NOT NULL, + "DB_NAME" character varying(128) DEFAULT NULL::character varying, + "TABLE_NAME" character varying(128) DEFAULT NULL::character varying, + "COLUMN_NAME" character varying(128) DEFAULT NULL::character varying, + "COLUMN_TYPE" character varying(128) DEFAULT NULL::character varying, + "TBL_ID" bigint NOT NULL, + "LONG_LOW_VALUE" bigint, + "LONG_HIGH_VALUE" bigint, + "DOUBLE_LOW_VALUE" double precision, + "DOUBLE_HIGH_VALUE" double precision, + "BIG_DECIMAL_LOW_VALUE" character varying(4000) DEFAULT NULL::character varying, + "BIG_DECIMAL_HIGH_VALUE" character varying(4000) DEFAULT NULL::character varying, + "NUM_NULLS" bigint NOT NULL, + "NUM_DISTINCTS" bigint, + "AVG_COL_LEN" double precision, + "MAX_COL_LEN" bigint, + "NUM_TRUES" bigint, + "NUM_FALSES" bigint, + "LAST_ANALYZED" bigint NOT NULL +); + +-- +-- Table structure for VERSION +-- +CREATE TABLE "VERSION" ( + "VER_ID" bigint, + "SCHEMA_VERSION" character varying(127) NOT NULL, + "VERSION_COMMENT" character varying(255) NOT NULL +); + +-- +-- Name: PART_COL_STATS Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "PART_COL_STATS" ( + "CS_ID" bigint NOT NULL, + "DB_NAME" character varying(128) DEFAULT NULL::character varying, + "TABLE_NAME" character varying(128) DEFAULT NULL::character varying, + "PARTITION_NAME" character varying(767) DEFAULT NULL::character varying, + "COLUMN_NAME" character varying(128) DEFAULT NULL::character varying, + "COLUMN_TYPE" character varying(128) DEFAULT NULL::character varying, + "PART_ID" bigint NOT NULL, + "LONG_LOW_VALUE" bigint, + "LONG_HIGH_VALUE" bigint, + "DOUBLE_LOW_VALUE" double precision, + "DOUBLE_HIGH_VALUE" double precision, + "BIG_DECIMAL_LOW_VALUE" character varying(4000) DEFAULT NULL::character varying, + "BIG_DECIMAL_HIGH_VALUE" character varying(4000) DEFAULT NULL::character varying, + "NUM_NULLS" bigint NOT NULL, + "NUM_DISTINCTS" bigint, + "AVG_COL_LEN" double precision, + "MAX_COL_LEN" bigint, + "NUM_TRUES" bigint, + "NUM_FALSES" bigint, + "LAST_ANALYZED" bigint NOT NULL +); + +-- +-- Table structure for FUNCS +-- +CREATE TABLE "FUNCS" ( + "FUNC_ID" BIGINT NOT NULL, + "CLASS_NAME" VARCHAR(4000), + "CREATE_TIME" INTEGER NOT NULL, + "DB_ID" BIGINT, + "FUNC_NAME" VARCHAR(128), + "FUNC_TYPE" INTEGER NOT NULL, + "OWNER_NAME" VARCHAR(128), + "OWNER_TYPE" VARCHAR(10), + PRIMARY KEY ("FUNC_ID") +); + +-- +-- Table structure for FUNC_RU +-- +CREATE TABLE "FUNC_RU" ( + "FUNC_ID" BIGINT NOT NULL, + "RESOURCE_TYPE" INTEGER NOT NULL, + "RESOURCE_URI" VARCHAR(4000), + "INTEGER_IDX" INTEGER NOT NULL, + PRIMARY KEY ("FUNC_ID", "INTEGER_IDX") +); + +-- +-- Name: BUCKETING_COLS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "BUCKETING_COLS" + ADD CONSTRAINT "BUCKETING_COLS_pkey" PRIMARY KEY ("SD_ID", "INTEGER_IDX"); + + +-- +-- Name: CDS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "CDS" + ADD CONSTRAINT "CDS_pkey" PRIMARY KEY ("CD_ID"); + + +-- +-- Name: COLUMNS_V2_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "COLUMNS_V2" + ADD CONSTRAINT "COLUMNS_V2_pkey" PRIMARY KEY ("CD_ID", "COLUMN_NAME"); + + +-- +-- Name: COLUMNS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "COLUMNS_OLD" + ADD CONSTRAINT "COLUMNS_pkey" PRIMARY KEY ("SD_ID", "COLUMN_NAME"); + + +-- +-- Name: DATABASE_PARAMS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "DATABASE_PARAMS" + ADD CONSTRAINT "DATABASE_PARAMS_pkey" PRIMARY KEY ("DB_ID", "PARAM_KEY"); + + +-- +-- Name: DBPRIVILEGEINDEX; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "DB_PRIVS" + ADD CONSTRAINT "DBPRIVILEGEINDEX" UNIQUE ("DB_ID", "PRINCIPAL_NAME", "PRINCIPAL_TYPE", "DB_PRIV", "GRANTOR", "GRANTOR_TYPE"); + + +-- +-- Name: DBS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "DBS" + ADD CONSTRAINT "DBS_pkey" PRIMARY KEY ("DB_ID"); + + +-- +-- Name: DB_PRIVS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "DB_PRIVS" + ADD CONSTRAINT "DB_PRIVS_pkey" PRIMARY KEY ("DB_GRANT_ID"); + + +-- +-- Name: GLOBALPRIVILEGEINDEX; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "GLOBAL_PRIVS" + ADD CONSTRAINT "GLOBALPRIVILEGEINDEX" UNIQUE ("PRINCIPAL_NAME", "PRINCIPAL_TYPE", "USER_PRIV", "GRANTOR", "GRANTOR_TYPE"); + + +-- +-- Name: GLOBAL_PRIVS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "GLOBAL_PRIVS" + ADD CONSTRAINT "GLOBAL_PRIVS_pkey" PRIMARY KEY ("USER_GRANT_ID"); + + +-- +-- Name: IDXS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "IDXS" + ADD CONSTRAINT "IDXS_pkey" PRIMARY KEY ("INDEX_ID"); + + +-- +-- Name: INDEX_PARAMS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "INDEX_PARAMS" + ADD CONSTRAINT "INDEX_PARAMS_pkey" PRIMARY KEY ("INDEX_ID", "PARAM_KEY"); + + +-- +-- Name: NUCLEUS_TABLES_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "NUCLEUS_TABLES" + ADD CONSTRAINT "NUCLEUS_TABLES_pkey" PRIMARY KEY ("CLASS_NAME"); + + +-- +-- Name: PARTITIONS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "PARTITIONS" + ADD CONSTRAINT "PARTITIONS_pkey" PRIMARY KEY ("PART_ID"); + + +-- +-- Name: PARTITION_EVENTS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "PARTITION_EVENTS" + ADD CONSTRAINT "PARTITION_EVENTS_pkey" PRIMARY KEY ("PART_NAME_ID"); + + +-- +-- Name: PARTITION_KEYS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "PARTITION_KEYS" + ADD CONSTRAINT "PARTITION_KEYS_pkey" PRIMARY KEY ("TBL_ID", "PKEY_NAME"); + + +-- +-- Name: PARTITION_KEY_VALS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "PARTITION_KEY_VALS" + ADD CONSTRAINT "PARTITION_KEY_VALS_pkey" PRIMARY KEY ("PART_ID", "INTEGER_IDX"); + + +-- +-- Name: PARTITION_PARAMS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "PARTITION_PARAMS" + ADD CONSTRAINT "PARTITION_PARAMS_pkey" PRIMARY KEY ("PART_ID", "PARAM_KEY"); + + +-- +-- Name: PART_COL_PRIVS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "PART_COL_PRIVS" + ADD CONSTRAINT "PART_COL_PRIVS_pkey" PRIMARY KEY ("PART_COLUMN_GRANT_ID"); + + +-- +-- Name: PART_PRIVS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "PART_PRIVS" + ADD CONSTRAINT "PART_PRIVS_pkey" PRIMARY KEY ("PART_GRANT_ID"); + + +-- +-- Name: ROLEENTITYINDEX; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "ROLES" + ADD CONSTRAINT "ROLEENTITYINDEX" UNIQUE ("ROLE_NAME"); + + +-- +-- Name: ROLES_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "ROLES" + ADD CONSTRAINT "ROLES_pkey" PRIMARY KEY ("ROLE_ID"); + + +-- +-- Name: ROLE_MAP_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "ROLE_MAP" + ADD CONSTRAINT "ROLE_MAP_pkey" PRIMARY KEY ("ROLE_GRANT_ID"); + + +-- +-- Name: SDS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "SDS" + ADD CONSTRAINT "SDS_pkey" PRIMARY KEY ("SD_ID"); + + +-- +-- Name: SD_PARAMS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "SD_PARAMS" + ADD CONSTRAINT "SD_PARAMS_pkey" PRIMARY KEY ("SD_ID", "PARAM_KEY"); + + +-- +-- Name: SEQUENCE_TABLE_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "SEQUENCE_TABLE" + ADD CONSTRAINT "SEQUENCE_TABLE_pkey" PRIMARY KEY ("SEQUENCE_NAME"); + + +-- +-- Name: SERDES_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "SERDES" + ADD CONSTRAINT "SERDES_pkey" PRIMARY KEY ("SERDE_ID"); + + +-- +-- Name: SERDE_PARAMS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "SERDE_PARAMS" + ADD CONSTRAINT "SERDE_PARAMS_pkey" PRIMARY KEY ("SERDE_ID", "PARAM_KEY"); + + +-- +-- Name: SORT_COLS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "SORT_COLS" + ADD CONSTRAINT "SORT_COLS_pkey" PRIMARY KEY ("SD_ID", "INTEGER_IDX"); + + +-- +-- Name: TABLE_PARAMS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "TABLE_PARAMS" + ADD CONSTRAINT "TABLE_PARAMS_pkey" PRIMARY KEY ("TBL_ID", "PARAM_KEY"); + + +-- +-- Name: TBLS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "TBLS" + ADD CONSTRAINT "TBLS_pkey" PRIMARY KEY ("TBL_ID"); + + +-- +-- Name: TBL_COL_PRIVS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "TBL_COL_PRIVS" + ADD CONSTRAINT "TBL_COL_PRIVS_pkey" PRIMARY KEY ("TBL_COLUMN_GRANT_ID"); + + +-- +-- Name: TBL_PRIVS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "TBL_PRIVS" + ADD CONSTRAINT "TBL_PRIVS_pkey" PRIMARY KEY ("TBL_GRANT_ID"); + + +-- +-- Name: TYPES_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "TYPES" + ADD CONSTRAINT "TYPES_pkey" PRIMARY KEY ("TYPES_ID"); + + +-- +-- Name: TYPE_FIELDS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "TYPE_FIELDS" + ADD CONSTRAINT "TYPE_FIELDS_pkey" PRIMARY KEY ("TYPE_NAME", "FIELD_NAME"); + +ALTER TABLE ONLY "SKEWED_STRING_LIST" + ADD CONSTRAINT "SKEWED_STRING_LIST_pkey" PRIMARY KEY ("STRING_LIST_ID"); + +ALTER TABLE ONLY "SKEWED_STRING_LIST_VALUES" + ADD CONSTRAINT "SKEWED_STRING_LIST_VALUES_pkey" PRIMARY KEY ("STRING_LIST_ID", "INTEGER_IDX"); + + +ALTER TABLE ONLY "SKEWED_COL_NAMES" + ADD CONSTRAINT "SKEWED_COL_NAMES_pkey" PRIMARY KEY ("SD_ID", "INTEGER_IDX"); + +ALTER TABLE ONLY "SKEWED_COL_VALUE_LOC_MAP" + ADD CONSTRAINT "SKEWED_COL_VALUE_LOC_MAP_pkey" PRIMARY KEY ("SD_ID", "STRING_LIST_ID_KID"); + +ALTER TABLE ONLY "SKEWED_VALUES" + ADD CONSTRAINT "SKEWED_VALUES_pkey" PRIMARY KEY ("SD_ID_OID", "INTEGER_IDX"); + +-- +-- Name: TAB_COL_STATS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- +ALTER TABLE ONLY "TAB_COL_STATS" ADD CONSTRAINT "TAB_COL_STATS_pkey" PRIMARY KEY("CS_ID"); + +-- +-- Name: PART_COL_STATS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- +ALTER TABLE ONLY "PART_COL_STATS" ADD CONSTRAINT "PART_COL_STATS_pkey" PRIMARY KEY("CS_ID"); + +-- +-- Name: UNIQUEINDEX; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "IDXS" + ADD CONSTRAINT "UNIQUEINDEX" UNIQUE ("INDEX_NAME", "ORIG_TBL_ID"); + + +-- +-- Name: UNIQUEPARTITION; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "PARTITIONS" + ADD CONSTRAINT "UNIQUEPARTITION" UNIQUE ("PART_NAME", "TBL_ID"); + + +-- +-- Name: UNIQUETABLE; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "TBLS" + ADD CONSTRAINT "UNIQUETABLE" UNIQUE ("TBL_NAME", "DB_ID"); + + +-- +-- Name: UNIQUE_DATABASE; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "DBS" + ADD CONSTRAINT "UNIQUE_DATABASE" UNIQUE ("NAME"); + + +-- +-- Name: UNIQUE_TYPE; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "TYPES" + ADD CONSTRAINT "UNIQUE_TYPE" UNIQUE ("TYPE_NAME"); + + +-- +-- Name: USERROLEMAPINDEX; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "ROLE_MAP" + ADD CONSTRAINT "USERROLEMAPINDEX" UNIQUE ("PRINCIPAL_NAME", "ROLE_ID", "GRANTOR", "GRANTOR_TYPE"); + + +-- +-- Name: BUCKETING_COLS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "BUCKETING_COLS_N49" ON "BUCKETING_COLS" USING btree ("SD_ID"); + + +-- +-- Name: COLUMNS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "COLUMNS_N49" ON "COLUMNS_OLD" USING btree ("SD_ID"); + + +-- +-- Name: DATABASE_PARAMS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "DATABASE_PARAMS_N49" ON "DATABASE_PARAMS" USING btree ("DB_ID"); + + +-- +-- Name: DB_PRIVS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "DB_PRIVS_N49" ON "DB_PRIVS" USING btree ("DB_ID"); + + +-- +-- Name: IDXS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "IDXS_N49" ON "IDXS" USING btree ("ORIG_TBL_ID"); + + +-- +-- Name: IDXS_N50; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "IDXS_N50" ON "IDXS" USING btree ("INDEX_TBL_ID"); + + +-- +-- Name: IDXS_N51; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "IDXS_N51" ON "IDXS" USING btree ("SD_ID"); + + +-- +-- Name: INDEX_PARAMS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "INDEX_PARAMS_N49" ON "INDEX_PARAMS" USING btree ("INDEX_ID"); + + +-- +-- Name: PARTITIONCOLUMNPRIVILEGEINDEX; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "PARTITIONCOLUMNPRIVILEGEINDEX" ON "PART_COL_PRIVS" USING btree ("PART_ID", "COLUMN_NAME", "PRINCIPAL_NAME", "PRINCIPAL_TYPE", "PART_COL_PRIV", "GRANTOR", "GRANTOR_TYPE"); + + +-- +-- Name: PARTITIONEVENTINDEX; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "PARTITIONEVENTINDEX" ON "PARTITION_EVENTS" USING btree ("PARTITION_NAME"); + + +-- +-- Name: PARTITIONS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "PARTITIONS_N49" ON "PARTITIONS" USING btree ("TBL_ID"); + + +-- +-- Name: PARTITIONS_N50; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "PARTITIONS_N50" ON "PARTITIONS" USING btree ("SD_ID"); + + +-- +-- Name: PARTITION_KEYS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "PARTITION_KEYS_N49" ON "PARTITION_KEYS" USING btree ("TBL_ID"); + + +-- +-- Name: PARTITION_KEY_VALS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "PARTITION_KEY_VALS_N49" ON "PARTITION_KEY_VALS" USING btree ("PART_ID"); + + +-- +-- Name: PARTITION_PARAMS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "PARTITION_PARAMS_N49" ON "PARTITION_PARAMS" USING btree ("PART_ID"); + + +-- +-- Name: PARTPRIVILEGEINDEX; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "PARTPRIVILEGEINDEX" ON "PART_PRIVS" USING btree ("PART_ID", "PRINCIPAL_NAME", "PRINCIPAL_TYPE", "PART_PRIV", "GRANTOR", "GRANTOR_TYPE"); + + +-- +-- Name: PART_COL_PRIVS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "PART_COL_PRIVS_N49" ON "PART_COL_PRIVS" USING btree ("PART_ID"); + + +-- +-- Name: PART_PRIVS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "PART_PRIVS_N49" ON "PART_PRIVS" USING btree ("PART_ID"); + + +-- +-- Name: PCS_STATS_IDX; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "PCS_STATS_IDX" ON "PART_COL_STATS" USING btree ("DB_NAME","TABLE_NAME","COLUMN_NAME","PARTITION_NAME"); + + +-- +-- Name: ROLE_MAP_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "ROLE_MAP_N49" ON "ROLE_MAP" USING btree ("ROLE_ID"); + + +-- +-- Name: SDS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "SDS_N49" ON "SDS" USING btree ("SERDE_ID"); + + +-- +-- Name: SD_PARAMS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "SD_PARAMS_N49" ON "SD_PARAMS" USING btree ("SD_ID"); + + +-- +-- Name: SERDE_PARAMS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "SERDE_PARAMS_N49" ON "SERDE_PARAMS" USING btree ("SERDE_ID"); + + +-- +-- Name: SORT_COLS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "SORT_COLS_N49" ON "SORT_COLS" USING btree ("SD_ID"); + + +-- +-- Name: TABLECOLUMNPRIVILEGEINDEX; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "TABLECOLUMNPRIVILEGEINDEX" ON "TBL_COL_PRIVS" USING btree ("TBL_ID", "COLUMN_NAME", "PRINCIPAL_NAME", "PRINCIPAL_TYPE", "TBL_COL_PRIV", "GRANTOR", "GRANTOR_TYPE"); + + +-- +-- Name: TABLEPRIVILEGEINDEX; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "TABLEPRIVILEGEINDEX" ON "TBL_PRIVS" USING btree ("TBL_ID", "PRINCIPAL_NAME", "PRINCIPAL_TYPE", "TBL_PRIV", "GRANTOR", "GRANTOR_TYPE"); + + +-- +-- Name: TABLE_PARAMS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "TABLE_PARAMS_N49" ON "TABLE_PARAMS" USING btree ("TBL_ID"); + + +-- +-- Name: TBLS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "TBLS_N49" ON "TBLS" USING btree ("DB_ID"); + + +-- +-- Name: TBLS_N50; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "TBLS_N50" ON "TBLS" USING btree ("SD_ID"); + + +-- +-- Name: TBL_COL_PRIVS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "TBL_COL_PRIVS_N49" ON "TBL_COL_PRIVS" USING btree ("TBL_ID"); + + +-- +-- Name: TBL_PRIVS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "TBL_PRIVS_N49" ON "TBL_PRIVS" USING btree ("TBL_ID"); + + +-- +-- Name: TYPE_FIELDS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "TYPE_FIELDS_N49" ON "TYPE_FIELDS" USING btree ("TYPE_NAME"); + +-- +-- Name: TAB_COL_STATS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "TAB_COL_STATS_N49" ON "TAB_COL_STATS" USING btree ("TBL_ID"); + +-- +-- Name: PART_COL_STATS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "PART_COL_STATS_N49" ON "PART_COL_STATS" USING btree ("PART_ID"); + +-- +-- Name: UNIQUEFUNCTION; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE UNIQUE INDEX "UNIQUEFUNCTION" ON "FUNCS" ("FUNC_NAME", "DB_ID"); + +-- +-- Name: FUNCS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "FUNCS_N49" ON "FUNCS" ("DB_ID"); + +-- +-- Name: FUNC_RU_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "FUNC_RU_N49" ON "FUNC_RU" ("FUNC_ID"); + + +ALTER TABLE ONLY "SKEWED_STRING_LIST_VALUES" + ADD CONSTRAINT "SKEWED_STRING_LIST_VALUES_fkey" FOREIGN KEY ("STRING_LIST_ID") REFERENCES "SKEWED_STRING_LIST"("STRING_LIST_ID") DEFERRABLE; + + +ALTER TABLE ONLY "SKEWED_COL_NAMES" + ADD CONSTRAINT "SKEWED_COL_NAMES_fkey" FOREIGN KEY ("SD_ID") REFERENCES "SDS"("SD_ID") DEFERRABLE; + + +ALTER TABLE ONLY "SKEWED_COL_VALUE_LOC_MAP" + ADD CONSTRAINT "SKEWED_COL_VALUE_LOC_MAP_fkey1" FOREIGN KEY ("SD_ID") REFERENCES "SDS"("SD_ID") DEFERRABLE; + +ALTER TABLE ONLY "SKEWED_COL_VALUE_LOC_MAP" + ADD CONSTRAINT "SKEWED_COL_VALUE_LOC_MAP_fkey2" FOREIGN KEY ("STRING_LIST_ID_KID") REFERENCES "SKEWED_STRING_LIST"("STRING_LIST_ID") DEFERRABLE; + +ALTER TABLE ONLY "SKEWED_VALUES" + ADD CONSTRAINT "SKEWED_VALUES_fkey1" FOREIGN KEY ("STRING_LIST_ID_EID") REFERENCES "SKEWED_STRING_LIST"("STRING_LIST_ID") DEFERRABLE; + +ALTER TABLE ONLY "SKEWED_VALUES" + ADD CONSTRAINT "SKEWED_VALUES_fkey2" FOREIGN KEY ("SD_ID_OID") REFERENCES "SDS"("SD_ID") DEFERRABLE; + + +-- +-- Name: BUCKETING_COLS_SD_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "BUCKETING_COLS" + ADD CONSTRAINT "BUCKETING_COLS_SD_ID_fkey" FOREIGN KEY ("SD_ID") REFERENCES "SDS"("SD_ID") DEFERRABLE; + + +-- +-- Name: COLUMNS_SD_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "COLUMNS_OLD" + ADD CONSTRAINT "COLUMNS_SD_ID_fkey" FOREIGN KEY ("SD_ID") REFERENCES "SDS"("SD_ID") DEFERRABLE; + + +-- +-- Name: COLUMNS_V2_CD_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "COLUMNS_V2" + ADD CONSTRAINT "COLUMNS_V2_CD_ID_fkey" FOREIGN KEY ("CD_ID") REFERENCES "CDS"("CD_ID") DEFERRABLE; + + +-- +-- Name: DATABASE_PARAMS_DB_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "DATABASE_PARAMS" + ADD CONSTRAINT "DATABASE_PARAMS_DB_ID_fkey" FOREIGN KEY ("DB_ID") REFERENCES "DBS"("DB_ID") DEFERRABLE; + + +-- +-- Name: DB_PRIVS_DB_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "DB_PRIVS" + ADD CONSTRAINT "DB_PRIVS_DB_ID_fkey" FOREIGN KEY ("DB_ID") REFERENCES "DBS"("DB_ID") DEFERRABLE; + + +-- +-- Name: IDXS_INDEX_TBL_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "IDXS" + ADD CONSTRAINT "IDXS_INDEX_TBL_ID_fkey" FOREIGN KEY ("INDEX_TBL_ID") REFERENCES "TBLS"("TBL_ID") DEFERRABLE; + + +-- +-- Name: IDXS_ORIG_TBL_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "IDXS" + ADD CONSTRAINT "IDXS_ORIG_TBL_ID_fkey" FOREIGN KEY ("ORIG_TBL_ID") REFERENCES "TBLS"("TBL_ID") DEFERRABLE; + + +-- +-- Name: IDXS_SD_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "IDXS" + ADD CONSTRAINT "IDXS_SD_ID_fkey" FOREIGN KEY ("SD_ID") REFERENCES "SDS"("SD_ID") DEFERRABLE; + + +-- +-- Name: INDEX_PARAMS_INDEX_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "INDEX_PARAMS" + ADD CONSTRAINT "INDEX_PARAMS_INDEX_ID_fkey" FOREIGN KEY ("INDEX_ID") REFERENCES "IDXS"("INDEX_ID") DEFERRABLE; + + +-- +-- Name: PARTITIONS_SD_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "PARTITIONS" + ADD CONSTRAINT "PARTITIONS_SD_ID_fkey" FOREIGN KEY ("SD_ID") REFERENCES "SDS"("SD_ID") DEFERRABLE; + + +-- +-- Name: PARTITIONS_TBL_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "PARTITIONS" + ADD CONSTRAINT "PARTITIONS_TBL_ID_fkey" FOREIGN KEY ("TBL_ID") REFERENCES "TBLS"("TBL_ID") DEFERRABLE; + + +-- +-- Name: PARTITION_KEYS_TBL_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "PARTITION_KEYS" + ADD CONSTRAINT "PARTITION_KEYS_TBL_ID_fkey" FOREIGN KEY ("TBL_ID") REFERENCES "TBLS"("TBL_ID") DEFERRABLE; + + +-- +-- Name: PARTITION_KEY_VALS_PART_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "PARTITION_KEY_VALS" + ADD CONSTRAINT "PARTITION_KEY_VALS_PART_ID_fkey" FOREIGN KEY ("PART_ID") REFERENCES "PARTITIONS"("PART_ID") DEFERRABLE; + + +-- +-- Name: PARTITION_PARAMS_PART_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "PARTITION_PARAMS" + ADD CONSTRAINT "PARTITION_PARAMS_PART_ID_fkey" FOREIGN KEY ("PART_ID") REFERENCES "PARTITIONS"("PART_ID") DEFERRABLE; + + +-- +-- Name: PART_COL_PRIVS_PART_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "PART_COL_PRIVS" + ADD CONSTRAINT "PART_COL_PRIVS_PART_ID_fkey" FOREIGN KEY ("PART_ID") REFERENCES "PARTITIONS"("PART_ID") DEFERRABLE; + + +-- +-- Name: PART_PRIVS_PART_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "PART_PRIVS" + ADD CONSTRAINT "PART_PRIVS_PART_ID_fkey" FOREIGN KEY ("PART_ID") REFERENCES "PARTITIONS"("PART_ID") DEFERRABLE; + + +-- +-- Name: ROLE_MAP_ROLE_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "ROLE_MAP" + ADD CONSTRAINT "ROLE_MAP_ROLE_ID_fkey" FOREIGN KEY ("ROLE_ID") REFERENCES "ROLES"("ROLE_ID") DEFERRABLE; + + +-- +-- Name: SDS_CD_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "SDS" + ADD CONSTRAINT "SDS_CD_ID_fkey" FOREIGN KEY ("CD_ID") REFERENCES "CDS"("CD_ID") DEFERRABLE; + + +-- +-- Name: SDS_SERDE_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "SDS" + ADD CONSTRAINT "SDS_SERDE_ID_fkey" FOREIGN KEY ("SERDE_ID") REFERENCES "SERDES"("SERDE_ID") DEFERRABLE; + + +-- +-- Name: SD_PARAMS_SD_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "SD_PARAMS" + ADD CONSTRAINT "SD_PARAMS_SD_ID_fkey" FOREIGN KEY ("SD_ID") REFERENCES "SDS"("SD_ID") DEFERRABLE; + + +-- +-- Name: SERDE_PARAMS_SERDE_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "SERDE_PARAMS" + ADD CONSTRAINT "SERDE_PARAMS_SERDE_ID_fkey" FOREIGN KEY ("SERDE_ID") REFERENCES "SERDES"("SERDE_ID") DEFERRABLE; + + +-- +-- Name: SORT_COLS_SD_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "SORT_COLS" + ADD CONSTRAINT "SORT_COLS_SD_ID_fkey" FOREIGN KEY ("SD_ID") REFERENCES "SDS"("SD_ID") DEFERRABLE; + + +-- +-- Name: TABLE_PARAMS_TBL_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "TABLE_PARAMS" + ADD CONSTRAINT "TABLE_PARAMS_TBL_ID_fkey" FOREIGN KEY ("TBL_ID") REFERENCES "TBLS"("TBL_ID") DEFERRABLE; + + +-- +-- Name: TBLS_DB_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "TBLS" + ADD CONSTRAINT "TBLS_DB_ID_fkey" FOREIGN KEY ("DB_ID") REFERENCES "DBS"("DB_ID") DEFERRABLE; + + +-- +-- Name: TBLS_SD_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "TBLS" + ADD CONSTRAINT "TBLS_SD_ID_fkey" FOREIGN KEY ("SD_ID") REFERENCES "SDS"("SD_ID") DEFERRABLE; + + +-- +-- Name: TBL_COL_PRIVS_TBL_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "TBL_COL_PRIVS" + ADD CONSTRAINT "TBL_COL_PRIVS_TBL_ID_fkey" FOREIGN KEY ("TBL_ID") REFERENCES "TBLS"("TBL_ID") DEFERRABLE; + + +-- +-- Name: TBL_PRIVS_TBL_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "TBL_PRIVS" + ADD CONSTRAINT "TBL_PRIVS_TBL_ID_fkey" FOREIGN KEY ("TBL_ID") REFERENCES "TBLS"("TBL_ID") DEFERRABLE; + + +-- +-- Name: TYPE_FIELDS_TYPE_NAME_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "TYPE_FIELDS" + ADD CONSTRAINT "TYPE_FIELDS_TYPE_NAME_fkey" FOREIGN KEY ("TYPE_NAME") REFERENCES "TYPES"("TYPES_ID") DEFERRABLE; + +-- +-- Name: TAB_COL_STATS_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- +ALTER TABLE ONLY "TAB_COL_STATS" ADD CONSTRAINT "TAB_COL_STATS_fkey" FOREIGN KEY("TBL_ID") REFERENCES "TBLS"("TBL_ID") DEFERRABLE; + + +-- +-- Name: PART_COL_STATS_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- +ALTER TABLE ONLY "PART_COL_STATS" ADD CONSTRAINT "PART_COL_STATS_fkey" FOREIGN KEY("PART_ID") REFERENCES "PARTITIONS"("PART_ID") DEFERRABLE; + + +ALTER TABLE ONLY "VERSION" ADD CONSTRAINT "VERSION_pkey" PRIMARY KEY ("VER_ID"); + +-- Name: FUNCS_FK1; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +ALTER TABLE ONLY "FUNCS" + ADD CONSTRAINT "FUNCS_FK1" FOREIGN KEY ("DB_ID") REFERENCES "DBS" ("DB_ID") DEFERRABLE; + +-- Name: FUNC_RU_FK1; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +ALTER TABLE ONLY "FUNC_RU" + ADD CONSTRAINT "FUNC_RU_FK1" FOREIGN KEY ("FUNC_ID") REFERENCES "FUNCS" ("FUNC_ID") DEFERRABLE; + +-- +-- Name: public; Type: ACL; Schema: -; Owner: hiveuser +-- + +REVOKE ALL ON SCHEMA public FROM PUBLIC; +GRANT ALL ON SCHEMA public TO PUBLIC; + +-- +-- PostgreSQL database dump complete +-- + +------------------------------ +-- Transaction and lock tables +------------------------------ +\i hive-txn-schema-0.13.0.postgres.sql; + +-- ----------------------------------------------------------------- +-- Record schema version. Should be the last step in the init script +-- ----------------------------------------------------------------- +INSERT INTO "VERSION" ("VER_ID", "SCHEMA_VERSION", "VERSION_COMMENT") VALUES (1, '0.15.0', 'Hive release version 0.15.0'); diff --git a/metastore/scripts/upgrade/postgres/hive-txn-schema-0.14.0.postgres.sql b/metastore/scripts/upgrade/postgres/hive-txn-schema-0.14.0.postgres.sql new file mode 100644 index 0000000..7676b8c --- /dev/null +++ b/metastore/scripts/upgrade/postgres/hive-txn-schema-0.14.0.postgres.sql @@ -0,0 +1,87 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- +-- Tables for transaction management +-- + +CREATE TABLE "txns" ( + "txn_id" bigint PRIMARY KEY, + "txn_state" char(1) NOT NULL, + "txn_started" bigint NOT NULL, + "txn_last_heartbeat" bigint NOT NULL, + "txn_user" varchar(128) NOT NULL, + "txn_host" varchar(128) NOT NULL +); + +CREATE TABLE "txn_components" ( + "tc_txnid" bigint REFERENCES "txns" ("txn_id"), + "tc_database" varchar(128) NOT NULL, + "tc_table" varchar(128), + "tc_partition" varchar(767) DEFAULT NULL +); + +CREATE TABLE "completed_txn_components" ( + "ctc_txnid" bigint, + "ctc_database" varchar(128) NOT NULL, + "ctc_table" varchar(128), + "ctc_partition" varchar(767) +); + +CREATE TABLE "next_txn_id" ( + "ntxn_next" bigint NOT NULL +); +INSERT INTO "next_txn_id" VALUES(1); + +CREATE TABLE "hive_locks" ( + "hl_lock_ext_id" bigint NOT NULL, + "hl_lock_int_id" bigint NOT NULL, + "hl_txnid" bigint, + "hl_db" varchar(128) NOT NULL, + "hl_table" varchar(128), + "hl_partition" varchar(767) DEFAULT NULL, + "hl_lock_state" char(1) NOT NULL, + "hl_lock_type" char(1) NOT NULL, + "hl_last_heartbeat" bigint NOT NULL, + "hl_acquired_at" bigint, + "hl_user" varchar(128) NOT NULL, + "hl_host" varchar(128) NOT NULL, + PRIMARY KEY("hl_lock_ext_id", "hl_lock_int_id") +); + +CREATE INDEX "hl_txnid_index" ON "hive_locks" USING hash ("hl_txnid"); + +CREATE TABLE "next_lock_id" ( + "nl_next" bigint NOT NULL +); +INSERT INTO "next_lock_id" VALUES(1); + +CREATE TABLE "compaction_queue" ( + "cq_id" bigint PRIMARY KEY, + "cq_database" varchar(128) NOT NULL, + "cq_table" varchar(128) NOT NULL, + "cq_partition" varchar(767), + "cq_state" char(1) NOT NULL, + "cq_type" char(1) NOT NULL, + "cq_worker_id" varchar(128), + "cq_start" bigint, + "cq_run_as" varchar(128) +); + +CREATE TABLE "next_compaction_queue_id" ( + "ncq_next" bigint NOT NULL +); +INSERT INTO "next_compaction_queue_id" VALUES(1); + diff --git a/metastore/scripts/upgrade/postgres/pre-0-upgrade-0.12.0-to-0.13.0.postgres.sql b/metastore/scripts/upgrade/postgres/pre-0-upgrade-0.12.0-to-0.13.0.postgres.sql new file mode 100644 index 0000000..bc028fb --- /dev/null +++ b/metastore/scripts/upgrade/postgres/pre-0-upgrade-0.12.0-to-0.13.0.postgres.sql @@ -0,0 +1 @@ +CREATE LANGUAGE plpgsql; diff --git a/metastore/scripts/upgrade/postgres/upgrade-0.14.0-to-0.15.0.postgres.sql b/metastore/scripts/upgrade/postgres/upgrade-0.14.0-to-0.15.0.postgres.sql new file mode 100644 index 0000000..ded09cd --- /dev/null +++ b/metastore/scripts/upgrade/postgres/upgrade-0.14.0-to-0.15.0.postgres.sql @@ -0,0 +1,11 @@ +SELECT 'Upgrading MetaStore schema from 0.14.0 to 0.15.0'; + +UPDATE "VERSION" SET "SCHEMA_VERSION"='0.15.0', "VERSION_COMMENT"='Hive release version 0.15.0' where "VER_ID"=1; +SELECT 'Finished upgrading MetaStore schema from 0.14.0 to 0.15.0'; + + +-- +-- Name: PCS_STATS_IDX; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + + diff --git a/metastore/scripts/upgrade/postgres/upgrade.order.postgres b/metastore/scripts/upgrade/postgres/upgrade.order.postgres index 682618a..509e52a 100644 --- a/metastore/scripts/upgrade/postgres/upgrade.order.postgres +++ b/metastore/scripts/upgrade/postgres/upgrade.order.postgres @@ -7,3 +7,4 @@ 0.11.0-to-0.12.0 0.12.0-to-0.13.0 0.13.0-to-0.14.0 +0.14.0-to-0.15.0 diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java index 963b184..fc6215a 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java @@ -354,7 +354,7 @@ public Partition alterPartition(final RawStore msdb, Warehouse wh, final String srcFs = wh.getFs(srcPath); destFs = wh.getFs(destPath); // check that src and dest are on the same file system - if (srcFs != destFs) { + if (!FileUtils.equalsFileSystem(srcFs, destFs)) { throw new InvalidOperationException("table new location " + destPath + " is on a different file system than the old location " + srcPath + ". This operation is not supported"); diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 47eca29..e3240ca 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -48,6 +48,8 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.regex.Pattern; +import javax.jdo.JDOException; + import org.apache.commons.cli.OptionBuilder; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -146,24 +148,30 @@ import org.apache.hadoop.hive.metastore.api.UnknownTableException; import org.apache.hadoop.hive.metastore.api.UnlockRequest; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; +import org.apache.hadoop.hive.metastore.events.AddIndexEvent; import org.apache.hadoop.hive.metastore.events.AddPartitionEvent; +import org.apache.hadoop.hive.metastore.events.AlterIndexEvent; import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent; import org.apache.hadoop.hive.metastore.events.AlterTableEvent; import org.apache.hadoop.hive.metastore.events.ConfigChangeEvent; import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent; import org.apache.hadoop.hive.metastore.events.CreateTableEvent; import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent; +import org.apache.hadoop.hive.metastore.events.DropIndexEvent; import org.apache.hadoop.hive.metastore.events.DropPartitionEvent; import org.apache.hadoop.hive.metastore.events.DropTableEvent; import org.apache.hadoop.hive.metastore.events.EventCleanerTask; import org.apache.hadoop.hive.metastore.events.LoadPartitionDoneEvent; +import org.apache.hadoop.hive.metastore.events.PreAddIndexEvent; import org.apache.hadoop.hive.metastore.events.PreAddPartitionEvent; +import org.apache.hadoop.hive.metastore.events.PreAlterIndexEvent; import org.apache.hadoop.hive.metastore.events.PreAlterPartitionEvent; import org.apache.hadoop.hive.metastore.events.PreAlterTableEvent; import org.apache.hadoop.hive.metastore.events.PreAuthorizationCallEvent; import org.apache.hadoop.hive.metastore.events.PreCreateDatabaseEvent; import org.apache.hadoop.hive.metastore.events.PreCreateTableEvent; import org.apache.hadoop.hive.metastore.events.PreDropDatabaseEvent; +import org.apache.hadoop.hive.metastore.events.PreDropIndexEvent; import org.apache.hadoop.hive.metastore.events.PreDropPartitionEvent; import org.apache.hadoop.hive.metastore.events.PreDropTableEvent; import org.apache.hadoop.hive.metastore.events.PreEventContext; @@ -184,6 +192,7 @@ import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge; +import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge.Server.ServerMode; import org.apache.hadoop.hive.thrift.TUGIContainingTransport; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ReflectionUtils; @@ -570,22 +579,50 @@ private void createDefaultDB_core(RawStore ms) throws MetaException, InvalidObje } /** - * create default database if it doesn't exist + * create default database if it doesn't exist. + * + * This is a potential contention when HiveServer2 using embedded metastore and Metastore + * Server try to concurrently invoke createDefaultDB. If one failed, JDOException was caught + * for one more time try, if failed again, simply ignored by warning, which meant another + * succeeds. * * @throws MetaException */ private void createDefaultDB() throws MetaException { try { createDefaultDB_core(getMS()); + } catch (JDOException e) { + LOG.warn("Retrying creating default database after error: " + e.getMessage(), e); + try { + createDefaultDB_core(getMS()); + } catch (InvalidObjectException e1) { + throw new MetaException(e1.getMessage()); + } } catch (InvalidObjectException e) { throw new MetaException(e.getMessage()); - } catch (MetaException e) { - throw e; } } - + /** + * create default roles if they don't exist. + * + * This is a potential contention when HiveServer2 using embedded metastore and Metastore + * Server try to concurrently invoke createDefaultRoles. If one failed, JDOException was caught + * for one more time try, if failed again, simply ignored by warning, which meant another + * succeeds. + * + * @throws MetaException + */ private void createDefaultRoles() throws MetaException { + try { + createDefaultRoles_core(); + } catch (JDOException e) { + LOG.warn("Retrying creating default roles after error: " + e.getMessage(), e); + createDefaultRoles_core(); + } + } + + private void createDefaultRoles_core() throws MetaException { RawStore ms = getMS(); try { @@ -622,7 +659,25 @@ private void createDefaultRoles() throws MetaException { } } + /** + * add admin users if they don't exist. + * + * This is a potential contention when HiveServer2 using embedded metastore and Metastore + * Server try to concurrently invoke addAdminUsers. If one failed, JDOException was caught for + * one more time try, if failed again, simply ignored by warning, which meant another succeeds. + * + * @throws MetaException + */ private void addAdminUsers() throws MetaException { + try { + addAdminUsers_core(); + } catch (JDOException e) { + LOG.warn("Retrying adding admin users after error: " + e.getMessage(), e); + addAdminUsers_core(); + } + } + + private void addAdminUsers_core() throws MetaException { // now add pre-configured users to admin role String userStr = HiveConf.getVar(hiveConf,ConfVars.USERS_IN_ADMIN_ROLE,"").trim(); @@ -1381,17 +1436,15 @@ private boolean is_table_exists(RawStore ms, String dbname, String name) return (ms.getTable(dbname, name) != null); } - private void drop_table_core(final RawStore ms, final String dbname, final String name, - final boolean deleteData, final EnvironmentContext envContext) - throws NoSuchObjectException, MetaException, IOException, - InvalidObjectException, InvalidInputException { + private boolean drop_table_core(final RawStore ms, final String dbname, final String name, + final boolean deleteData, final EnvironmentContext envContext, + final String indexName) throws NoSuchObjectException, + MetaException, IOException, InvalidObjectException, InvalidInputException { boolean success = false; boolean isExternal = false; Path tblPath = null; List partPaths = null; Table tbl = null; - isExternal = false; - boolean isIndexTable = false; try { ms.openTransaction(); // drop any partitions @@ -1405,8 +1458,8 @@ private void drop_table_core(final RawStore ms, final String dbname, final Strin firePreEvent(new PreDropTableEvent(tbl, deleteData, this)); - isIndexTable = isIndexTable(tbl); - if (isIndexTable) { + boolean isIndexTable = isIndexTable(tbl); + if (indexName == null && isIndexTable) { throw new RuntimeException( "The table " + name + " is an index table. Please do drop index instead."); } @@ -1428,7 +1481,8 @@ private void drop_table_core(final RawStore ms, final String dbname, final Strin if (tbl.getSd().getLocation() != null) { tblPath = new Path(tbl.getSd().getLocation()); if (!wh.isWritable(tblPath.getParent())) { - throw new MetaException("Table metadata not deleted since " + + String target = indexName == null ? "Table" : "Index table"; + throw new MetaException(target + " metadata not deleted since " + tblPath.getParent() + " is not writable by " + hiveConf.getUser()); } @@ -1439,17 +1493,17 @@ private void drop_table_core(final RawStore ms, final String dbname, final Strin tbl.getPartitionKeys(), deleteData && !isExternal); if (!ms.dropTable(dbname, name)) { - throw new MetaException("Unable to drop table"); + String tableName = dbname + "." + name; + throw new MetaException(indexName == null ? "Unable to drop table " + tableName: + "Unable to drop index table " + tableName + " for index " + indexName); } success = ms.commitTransaction(); } finally { if (!success) { ms.rollbackTransaction(); } else if (deleteData && !isExternal) { - boolean ifPurge = false; - if (envContext != null){ - ifPurge = Boolean.parseBoolean(envContext.getProperties().get("ifPurge")); - } + boolean ifPurge = envContext != null && + Boolean.parseBoolean(envContext.getProperties().get("ifPurge")); // Delete the data in the partitions which have other locations deletePartitionData(partPaths, ifPurge); // Delete the data in the table @@ -1462,6 +1516,7 @@ private void drop_table_core(final RawStore ms, final String dbname, final Strin listener.onDropTable(dropTableEvent); } } + return success; } /** @@ -1605,8 +1660,7 @@ public void drop_table_with_environment_context(final String dbname, final Strin boolean success = false; Exception ex = null; try { - drop_table_core(getMS(), dbname, name, deleteData, envContext); - success = true; + success = drop_table_core(getMS(), dbname, name, deleteData, envContext, null); } catch (IOException e) { ex = e; throw new MetaException(e.getMessage()); @@ -3161,7 +3215,12 @@ public void alter_index(final String dbname, final String base_table_name, boolean success = false; Exception ex = null; + Index oldIndex = null; try { + oldIndex = get_index_by_name(dbname, base_table_name, index_name); + + firePreEvent(new PreAlterIndexEvent(oldIndex, newIndex, this)); + getMS().alterIndex(dbname, base_table_name, index_name, newIndex); success = true; } catch (InvalidObjectException e) { @@ -3178,6 +3237,10 @@ public void alter_index(final String dbname, final String base_table_name, } } finally { endFunction("alter_index", success, ex, base_table_name); + for (MetaStoreEventListener listener : listeners) { + AlterIndexEvent alterIndexEvent = new AlterIndexEvent(oldIndex, newIndex, success, this); + listener.onAlterIndex(alterIndexEvent); + } } return; } @@ -3307,7 +3370,7 @@ public void alter_table_with_environment_context(final String dbname, ret = tbl.getSd().getCols(); } else { try { - Deserializer s = MetaStoreUtils.getDeserializer(hiveConf, tbl); + Deserializer s = MetaStoreUtils.getDeserializer(hiveConf, tbl, false); ret = MetaStoreUtils.getFieldsFromDeserializer(tableName, s); } catch (SerDeException e) { StringUtils.stringifyException(e); @@ -3723,6 +3786,8 @@ private Index add_index_core(final RawStore ms, final Index index, final Table i try { ms.openTransaction(); + firePreEvent(new PreAddIndexEvent(index, this)); + Index old_index = null; try { old_index = get_index_by_name(index.getDbName(), index @@ -3770,6 +3835,10 @@ private Index add_index_core(final RawStore ms, final Index index, final Table i } ms.rollbackTransaction(); } + for (MetaStoreEventListener listener : listeners) { + AddIndexEvent addIndexEvent = new AddIndexEvent(index, success, this); + listener.onAddIndex(addIndexEvent); + } } } @@ -3804,16 +3873,17 @@ private boolean drop_index_by_name_core(final RawStore ms, MetaException, TException, IOException, InvalidObjectException, InvalidInputException { boolean success = false; + Index index = null; Path tblPath = null; List partPaths = null; try { ms.openTransaction(); // drop the underlying index table - Index index = get_index_by_name(dbName, tblName, indexName); - if (index == null) { - throw new NoSuchObjectException(indexName + " doesn't exist"); - } + index = get_index_by_name(dbName, tblName, indexName); // throws exception if not exists + + firePreEvent(new PreDropIndexEvent(index, this)); + ms.dropIndex(dbName, tblName, indexName); String idxTblName = index.getIndexTableName(); @@ -3834,26 +3904,29 @@ private boolean drop_index_by_name_core(final RawStore ms, } // Drop the partitions and get a list of partition locations which need to be deleted - partPaths = dropPartitionsAndGetLocations(ms, dbName, idxTblName, tblPath, + partPaths = dropPartitionsAndGetLocations(ms, qualified[0], qualified[1], tblPath, tbl.getPartitionKeys(), deleteData); - if (!ms.dropTable(dbName, idxTblName)) { + if (!ms.dropTable(qualified[0], qualified[1])) { throw new MetaException("Unable to drop underlying data table " - + idxTblName + " for index " + idxTblName); + + qualified[0] + "." + qualified[1] + " for index " + indexName); } } success = ms.commitTransaction(); } finally { if (!success) { ms.rollbackTransaction(); - return false; } else if (deleteData && tblPath != null) { deletePartitionData(partPaths); deleteTableData(tblPath); // ok even if the data is not deleted } + for (MetaStoreEventListener listener : listeners) { + DropIndexEvent dropIndexEvent = new DropIndexEvent(index, success, this); + listener.onDropIndex(dropIndexEvent); + } } - return true; + return success; } @Override @@ -3872,7 +3945,7 @@ public Index get_index_by_name(final String dbName, final String tblName, ex = e; rethrowException(e); } finally { - endFunction("drop_index_by_name", ret != null, ex, tblName); + endFunction("get_index_by_name", ret != null, ex, tblName); } return ret; } @@ -5489,13 +5562,20 @@ public boolean set_aggr_stats_for(SetPartitionsStatsRequest request) } - public static IHMSHandler newHMSHandler(String name, HiveConf hiveConf) throws MetaException { - return newHMSHandler(name, hiveConf, false); + public static IHMSHandler newRetryingHMSHandler(IHMSHandler baseHandler, HiveConf hiveConf) + throws MetaException { + return newRetryingHMSHandler(baseHandler, hiveConf, false); + } + + public static IHMSHandler newRetryingHMSHandler(IHMSHandler baseHandler, HiveConf hiveConf, + boolean local) throws MetaException { + return RetryingHMSHandler.getProxy(hiveConf, baseHandler, local); } - public static IHMSHandler newHMSHandler(String name, HiveConf hiveConf, boolean local) + public static Iface newRetryingHMSHandler(String name, HiveConf conf, boolean local) throws MetaException { - return RetryingHMSHandler.getProxy(hiveConf, name, local); + HMSHandler baseHandler = new HiveMetaStore.HMSHandler(name, conf, false); + return RetryingHMSHandler.getProxy(conf, baseHandler, local); } /** @@ -5691,7 +5771,7 @@ public static void startMetaStore(int port, HadoopThriftAuthBridge bridge, try { isMetaStoreRemote = true; // Server will create new threads up to max as necessary. After an idle - // period, it will destory threads to keep the number of threads in the + // period, it will destroy threads to keep the number of threads in the // pool to min. int minWorkerThreads = conf.getIntVar(HiveConf.ConfVars.METASTORESERVERMINTHREADS); int maxWorkerThreads = conf.getIntVar(HiveConf.ConfVars.METASTORESERVERMAXTHREADS); @@ -5704,6 +5784,9 @@ public static void startMetaStore(int port, HadoopThriftAuthBridge bridge, TProcessor processor; TTransportFactory transFactory; + HMSHandler baseHandler = new HiveMetaStore.HMSHandler("new db based metaserver", conf, + false); + IHMSHandler handler = newRetryingHMSHandler(baseHandler, conf); if (useSasl) { // we are in secure mode. if (useFramedTransport) { @@ -5713,17 +5796,14 @@ public static void startMetaStore(int port, HadoopThriftAuthBridge bridge, conf.getVar(HiveConf.ConfVars.METASTORE_KERBEROS_KEYTAB_FILE), conf.getVar(HiveConf.ConfVars.METASTORE_KERBEROS_PRINCIPAL)); // start delegation token manager - HMSHandler hmsHandler = new HMSHandler("new db based metaserver", conf); - saslServer.startDelegationTokenSecretManager(conf, hmsHandler); + saslServer.startDelegationTokenSecretManager(conf, baseHandler.getMS(), ServerMode.METASTORE); transFactory = saslServer.createTransportFactory( MetaStoreUtils.getMetaStoreSaslProperties(conf)); processor = saslServer.wrapProcessor( - new ThriftHiveMetastore.Processor(hmsHandler)); + new ThriftHiveMetastore.Processor(handler)); LOG.info("Starting DB backed MetaStore Server in Secure Mode"); } else { // we are in unsecure mode. - IHMSHandler handler = newHMSHandler("new db based metaserver", conf); - if (conf.getBoolVar(ConfVars.METASTORE_EXECUTE_SET_UGI)) { transFactory = useFramedTransport ? new ChainedTTransportFactory(new TFramedTransport.Factory(), @@ -5878,7 +5958,7 @@ private static void initializeAndStartThread(MetaStoreThread thread, HiveConf co LOG.info("Starting metastore thread of type " + thread.getClass().getName()); thread.setHiveConf(conf); thread.setThreadId(nextThreadId++); - thread.init(new MetaStoreThread.BooleanPointer()); + thread.init(new MetaStoreThread.BooleanPointer(), new MetaStoreThread.BooleanPointer()); thread.start(); } } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index c25f13e..9f2b4fc 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -175,7 +175,7 @@ public HiveMetaStoreClient(HiveConf conf, HiveMetaHookLoader hookLoader) if (localMetaStore) { // instantiate the metastore server handler directly instead of connecting // through the network - client = HiveMetaStore.newHMSHandler("hive client", conf, true); + client = HiveMetaStore.newRetryingHMSHandler("hive client", conf, true); isConnected = true; snapshotActiveConf(); return; diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java index 5a8591a..dea0f91 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java @@ -45,10 +45,13 @@ import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; +import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.PrincipalType; import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.SkewedInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; @@ -143,13 +146,18 @@ public boolean isCompatibleDatastore() { } /** - * See {@link #trySetAnsiQuotesForMysql()}. + * This function is intended to be called by functions before they put together a query + * Thus, any query-specific instantiation to be done from within the transaction is done + * here - for eg., for MySQL, we signal that we want to use ANSI SQL quoting behaviour */ - private void setAnsiQuotesForMysql() throws MetaException { - try { - trySetAnsiQuotesForMysql(); - } catch (SQLException sqlEx) { - throw new MetaException("Error setting ansi quotes: " + sqlEx.getMessage()); + private void doDbSpecificInitializationsBeforeQuery() throws MetaException { + if (isMySql){ + try { + assert pm.currentTransaction().isActive(); // must be inside tx together with queries + trySetAnsiQuotesForMysql(); + } catch (SQLException sqlEx) { + throw new MetaException("Error setting ansi quotes: " + sqlEx.getMessage()); + } } } @@ -171,6 +179,78 @@ private void trySetAnsiQuotesForMysql() throws SQLException { } } + public Database getDatabase(String dbName) throws MetaException{ + Query queryDbSelector = null; + Query queryDbParams = null; + try { + dbName = dbName.toLowerCase(); + + doDbSpecificInitializationsBeforeQuery(); + + String queryTextDbSelector= "select " + + "\"DB_ID\", \"NAME\", \"DB_LOCATION_URI\", \"DESC\", " + + "\"OWNER_NAME\", \"OWNER_TYPE\" " + + "FROM \"DBS\" where \"NAME\" = ? "; + Object[] params = new Object[] { dbName }; + queryDbSelector = pm.newQuery("javax.jdo.query.SQL", queryTextDbSelector); + + LOG.debug("getDatabase:query instantiated : " + queryTextDbSelector + " with param ["+params[0]+"]"); + + List sqlResult = (List)queryDbSelector.executeWithArray(params); + if ((sqlResult == null) || sqlResult.isEmpty()) { + LOG.debug("getDatabase:queryDbSelector ran, returned no/empty results, returning NoSuchObjectException"); + throw new MetaException("There is no database named " + dbName); + } + + assert(sqlResult.size() == 1); + if (sqlResult.get(0) == null){ + LOG.debug("getDatabase:queryDbSelector ran, returned results, but the result entry was null, returning NoSuchObjectException"); + throw new MetaException("There is no database named " + dbName); + } + + Object[] dbline = sqlResult.get(0); + Long dbid = StatObjectConverter.extractSqlLong(dbline[0]); + + String queryTextDbParams = "select \"PARAM_KEY\", \"PARAM_VALUE\" " + + " FROM \"DATABASE_PARAMS\" " + + " WHERE \"DB_ID\" = ? " + + " AND \"PARAM_KEY\" IS NOT NULL"; + Object[] params2 = new Object[] { dbid }; + queryDbParams = pm.newQuery("javax.jdo.query.SQL",queryTextDbParams); + LOG.debug("getDatabase:query2 instantiated : " + queryTextDbParams + " with param ["+params2[0]+"]"); + + Map dbParams = new HashMap(); + List sqlResult2 = ensureList(queryDbParams.executeWithArray(params2)); + if (!sqlResult2.isEmpty()){ + for (Object[] line : sqlResult2){ + dbParams.put(extractSqlString(line[0]),extractSqlString(line[1])); + } + } + LOG.debug("getDatabase: instantiating db object to return"); + Database db = new Database(); + db.setName(extractSqlString(dbline[1])); + db.setLocationUri(extractSqlString(dbline[2])); + db.setDescription(extractSqlString(dbline[3])); + db.setOwnerName(extractSqlString(dbline[4])); + String type = extractSqlString(dbline[5]); + db.setOwnerType((null == type || type.trim().isEmpty()) ? null : PrincipalType.valueOf(type)); + db.setParameters(dbParams); + if (LOG.isDebugEnabled()){ + LOG.debug("getDatabase: directsql returning db " + db.getName() + + " locn["+db.getLocationUri() +"] desc [" +db.getDescription() + + "] owner [" + db.getOwnerName() + "] ownertype ["+ db.getOwnerType() +"]"); + } + return db; + } finally { + if (queryDbSelector != null){ + queryDbSelector.closeAll(); + } + if (queryDbParams != null){ + queryDbParams.closeAll(); + } + } + } + /** * Gets partitions by using direct SQL queries. * @param dbName Metastore db name. @@ -260,10 +340,8 @@ private boolean isViewTable(String dbName, String tblName) throws MetaException tblName = tblName.toLowerCase(); // We have to be mindful of order during filtering if we are not returning all partitions. String orderForFilter = (max != null) ? " order by \"PART_NAME\" asc" : ""; - if (isMySql) { - assert pm.currentTransaction().isActive(); - setAnsiQuotesForMysql(); // must be inside tx together with queries - } + + doDbSpecificInitializationsBeforeQuery(); // Get all simple fields for partitions and related objects, which we can map one-on-one. // We will do this in 2 queries to use different existing indices for each one. @@ -621,6 +699,11 @@ private int extractSqlInt(Object field) { return ((Number)field).intValue(); } + private String extractSqlString(Object value) { + if (value == null) return null; + return value.toString(); + } + private static String trimCommaList(StringBuilder sb) { if (sb.length() > 0) { sb.setLength(sb.length() - 1); diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java index ec1dca2..792ef42 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java @@ -21,6 +21,8 @@ import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.events.AddIndexEvent; +import org.apache.hadoop.hive.metastore.events.AlterIndexEvent; import org.apache.hadoop.hive.metastore.events.AddPartitionEvent; import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent; import org.apache.hadoop.hive.metastore.events.AlterTableEvent; @@ -28,6 +30,7 @@ import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent; import org.apache.hadoop.hive.metastore.events.CreateTableEvent; import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent; +import org.apache.hadoop.hive.metastore.events.DropIndexEvent; import org.apache.hadoop.hive.metastore.events.DropPartitionEvent; import org.apache.hadoop.hive.metastore.events.DropTableEvent; import org.apache.hadoop.hive.metastore.events.LoadPartitionDoneEvent; @@ -117,7 +120,27 @@ public void onDropDatabase (DropDatabaseEvent dbEvent) throws MetaException { * @throws MetaException */ public void onLoadPartitionDone(LoadPartitionDoneEvent partSetDoneEvent) throws MetaException { + } + + /** + * @param indexEvent index event + * @throws MetaException + */ + public void onAddIndex(AddIndexEvent indexEvent) throws MetaException { + } + + /** + * @param indexEvent index event + * @throws MetaException + */ + public void onDropIndex(DropIndexEvent indexEvent) throws MetaException { + } + /** + * @param indexEvent index event + * @throws MetaException + */ + public void onAlterIndex(AlterIndexEvent indexEvent) throws MetaException { } @Override diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInit.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInit.java index f674244..4a139a9 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInit.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInit.java @@ -41,25 +41,30 @@ } /** - * Updates the connection URL in hiveConf using the hook - * + * Updates the connection URL in hiveConf using the hook (if a hook has been + * set using hive.metastore.ds.connection.url.hook property) + * @param originalConf - original configuration used to look up hook settings + * @param activeConf - the configuration file in use for looking up db url + * @param badUrl + * @param updateData - hook information * @return true if a new connection URL was loaded into the thread local * configuration + * @throws MetaException */ - static boolean updateConnectionURL(HiveConf hiveConf, Configuration conf, + static boolean updateConnectionURL(HiveConf originalConf, Configuration activeConf, String badUrl, MetaStoreInitData updateData) throws MetaException { String connectUrl = null; - String currentUrl = MetaStoreInit.getConnectionURL(conf); + String currentUrl = MetaStoreInit.getConnectionURL(activeConf); try { // We always call init because the hook name in the configuration could // have changed. - MetaStoreInit.initConnectionUrlHook(hiveConf, updateData); + MetaStoreInit.initConnectionUrlHook(originalConf, updateData); if (updateData.urlHook != null) { if (badUrl != null) { updateData.urlHook.notifyBadConnectionUrl(badUrl); } - connectUrl = updateData.urlHook.getJdoConnectionUrl(hiveConf); + connectUrl = updateData.urlHook.getJdoConnectionUrl(originalConf); } } catch (Exception e) { LOG.error("Exception while getting connection URL from the hook: " + @@ -71,7 +76,7 @@ static boolean updateConnectionURL(HiveConf hiveConf, Configuration conf, String.format("Overriding %s with %s", HiveConf.ConfVars.METASTORECONNECTURLKEY.toString(), connectUrl)); - conf.set(HiveConf.ConfVars.METASTORECONNECTURLKEY.toString(), + activeConf.set(HiveConf.ConfVars.METASTORECONNECTURLKEY.toString(), connectUrl); return true; } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java index af56ef6..50d03a8 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java @@ -37,6 +37,7 @@ private static String UPGRADE_FILE_PREFIX="upgrade-"; private static String INIT_FILE_PREFIX="hive-schema-"; private static String VERSION_UPGRADE_LIST = "upgrade.order"; + private static String PRE_UPGRADE_PREFIX = "pre-"; private final String dbType; private final String hiveSchemaVersions[]; private final HiveConf hiveConf; @@ -138,6 +139,10 @@ private String generateUpgradeFileName(String fileVersion) { return UPGRADE_FILE_PREFIX + fileVersion + "." + dbType + SQL_FILE_EXTENSION; } + public static String getPreUpgradeScriptName(int index, String upgradeScriptName) { + return PRE_UPGRADE_PREFIX + index + "-" + upgradeScriptName; + } + public static String getHiveSchemaVersion() { String hiveVersion = HiveVersionInfo.getShortVersion(); // if there is an equivalent version, return that, else return this version @@ -149,4 +154,4 @@ public static String getHiveSchemaVersion() { } } -} +} \ No newline at end of file diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreThread.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreThread.java index 6e18a5b..ff265e5 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreThread.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreThread.java @@ -43,8 +43,13 @@ * have been called. * @param stop a flag to watch for when to stop. If this value is set to true, * the thread will terminate the next time through its main loop. + * @param looped a flag that is set to true everytime a thread goes through it's main loop. + * This is purely for testing so that tests can assure themselves that the thread + * has run through it's loop once. The test can set this value to false. The + * thread should then assure that the loop has been gone completely through at + * least once. */ - void init(BooleanPointer stop) throws MetaException; + void init(BooleanPointer stop, BooleanPointer looped) throws MetaException; /** * Run the thread in the background. This must not be called until diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java index 25c180d..1ac5aff 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java @@ -357,15 +357,21 @@ public static boolean updatePartitionStatsFast(PartitionSpecProxy.PartitionItera * */ static public Deserializer getDeserializer(Configuration conf, - org.apache.hadoop.hive.metastore.api.Table table) throws MetaException { + org.apache.hadoop.hive.metastore.api.Table table, boolean skipConfError) throws + MetaException { String lib = table.getSd().getSerdeInfo().getSerializationLib(); if (lib == null) { return null; } try { Deserializer deserializer = ReflectionUtils.newInstance(conf.getClassByName(lib). - asSubclass(Deserializer.class), conf); - SerDeUtils.initializeSerDe(deserializer, conf, MetaStoreUtils.getTableMetadata(table), null); + asSubclass(Deserializer.class), conf); + if (skipConfError) { + SerDeUtils.initializeSerDeWithoutErrorCheck(deserializer, conf, + MetaStoreUtils.getTableMetadata(table), null); + } else { + SerDeUtils.initializeSerDe(deserializer, conf, MetaStoreUtils.getTableMetadata(table), null); + } return deserializer; } catch (RuntimeException e) { throw e; @@ -376,6 +382,12 @@ static public Deserializer getDeserializer(Configuration conf, } } + public static Class getDeserializerClass( + Configuration conf, org.apache.hadoop.hive.metastore.api.Table table) throws Exception { + String lib = table.getSd().getSerdeInfo().getSerializationLib(); + return lib == null ? null : conf.getClassByName(lib).asSubclass(Deserializer.class); + } + /** * getDeserializer * diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 637a39a..291f3be 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -64,7 +64,6 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.api.AggrStats; -import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; @@ -92,7 +91,6 @@ import org.apache.hadoop.hive.metastore.api.ResourceUri; import org.apache.hadoop.hive.metastore.api.Role; import org.apache.hadoop.hive.metastore.api.SerDeInfo; -import org.apache.hadoop.hive.metastore.api.SetPartitionsStatsRequest; import org.apache.hadoop.hive.metastore.api.SkewedInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; @@ -197,7 +195,7 @@ public Configuration getConf() { } /** - * Called whenever this object is instantiated using ReflectionUils, and also + * Called whenever this object is instantiated using ReflectionUtils, and also * on connection retries. In cases of connection retries, conf will usually * contain modified values. */ @@ -523,6 +521,34 @@ private MDatabase getMDatabase(String name) throws NoSuchObjectException { @Override public Database getDatabase(String name) throws NoSuchObjectException { + try { + return getDatabaseInternal(name); + } catch (MetaException e) { + // Signature restriction to NSOE, and NSOE being a flat exception prevents us from + // setting the cause of the NSOE as the MetaException. We should not lose the info + // we got here, but it's very likely that the MetaException is irrelevant and is + // actually an NSOE message, so we should log it and throw an NSOE with the msg. + LOG.warn("Got a MetaException trying to call getDatabase(" + +name+"), returning NoSuchObjectException", e); + throw new NoSuchObjectException(e.getMessage()); + } + } + + public Database getDatabaseInternal(String name) throws MetaException, NoSuchObjectException { + return new GetDbHelper(name, null, true, true) { + @Override + protected Database getSqlResult(GetHelper ctx) throws MetaException { + return directSql.getDatabase(dbName); + } + + @Override + protected Database getJdoResult(GetHelper ctx) throws MetaException, NoSuchObjectException { + return getJDODatabase(dbName); + } + }.run(false); + } + + public Database getJDODatabase(String name) throws NoSuchObjectException { MDatabase mdb = null; boolean commited = false; try { @@ -2282,7 +2308,14 @@ public GetHelper(String dbName, String tblName, boolean allowSql, boolean allowJ assert allowSql || allowJdo; this.allowJdo = allowJdo; this.dbName = dbName.toLowerCase(); - this.tblName = tblName.toLowerCase(); + if (tblName != null){ + this.tblName = tblName.toLowerCase(); + } else { + // tblName can be null in cases of Helper being used at a higher + // abstraction level, such as with datbases + this.tblName = null; + this.table = null; + } this.doTrace = LOG.isDebugEnabled(); this.isInTxn = isActiveTransaction(); @@ -2331,7 +2364,7 @@ public T run(boolean initTable) throws MetaException, NoSuchObjectException { private void start(boolean initTable) throws MetaException, NoSuchObjectException { start = doTrace ? System.nanoTime() : 0; openTransaction(); - if (initTable) { + if (initTable && (tblName != null)) { table = ensureGetTable(dbName, tblName); } } @@ -2398,6 +2431,27 @@ protected String describeResult() { } } + private abstract class GetDbHelper extends GetHelper { + /** + * GetHelper for returning db info using directSql/JDO. + * Since this is a db-level call, tblName is ignored, and null is passed irrespective of what is passed in. + * @param dbName The Database Name + * @param tblName Placeholder param to match signature, always ignored. + * @param allowSql Whether or not we allow DirectSQL to perform this query. + * @param allowJdo Whether or not we allow ORM to perform this query. + * @throws MetaException + */ + public GetDbHelper( + String dbName, String tblName, boolean allowSql, boolean allowJdo) throws MetaException { + super(dbName,null,allowSql,allowJdo); + } + + @Override + protected String describeResult() { + return "db details for db " + dbName; + } + } + private abstract class GetStatHelper extends GetHelper { public GetStatHelper( String dbName, String tblName, boolean allowSql, boolean allowJdo) throws MetaException { @@ -2665,7 +2719,7 @@ public void alterTable(String dbname, String name, Table newTable) throw new MetaException("table " + name + " doesn't exist"); } - // For now only alter name, owner, paramters, cols, bucketcols are allowed + // For now only alter name, owner, parameters, cols, bucketcols are allowed oldt.setDatabase(newt.getDatabase()); oldt.setTableName(newt.getTableName().toLowerCase()); oldt.setParameters(newt.getParameters()); @@ -2708,7 +2762,7 @@ public void alterIndex(String dbname, String baseTblName, String name, Index new throw new MetaException("index " + name + " doesn't exist"); } - // For now only alter paramters are allowed + // For now only alter parameters are allowed oldi.setParameters(newi.getParameters()); // commit the changes @@ -2878,7 +2932,7 @@ private void preDropStorageDescriptor(MStorageDescriptor msd) { MColumnDescriptor mcd = msd.getCD(); // Because there is a 1-N relationship between CDs and SDs, // we must set the SD's CD to null first before dropping the storage descriptor - // to satisfy foriegn key constraints. + // to satisfy foreign key constraints. msd.setCD(null); removeUnusedColumnDescriptor(mcd); } @@ -3019,19 +3073,26 @@ public Index getIndex(String dbName, String origTableName, String indexName) } private Index convertToIndex(MIndex mIndex) throws MetaException { - if(mIndex == null) { + if (mIndex == null) { return null; } + MTable origTable = mIndex.getOrigTable(); + MTable indexTable = mIndex.getIndexTable(); + + String[] qualified = MetaStoreUtils.getQualifiedName( + origTable.getDatabase().getName(), indexTable.getTableName()); + String indexTableName = qualified[0] + "." + qualified[1]; + return new Index( mIndex.getIndexName(), mIndex.getIndexHandlerClass(), - mIndex.getOrigTable().getDatabase().getName(), - mIndex.getOrigTable().getTableName(), + origTable.getDatabase().getName(), + origTable.getTableName(), mIndex.getCreateTime(), mIndex.getLastAccessTime(), - mIndex.getIndexTable().getTableName(), - this.convertToStorageDescriptor(mIndex.getSd()), + indexTableName, + convertToStorageDescriptor(mIndex.getSd()), mIndex.getParameters(), mIndex.getDeferredRebuild()); diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java index 63a113c..b18eb21 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java @@ -41,34 +41,34 @@ private static final Log LOG = LogFactory.getLog(RetryingHMSHandler.class); - private final IHMSHandler base; + private final IHMSHandler baseHandler; private final MetaStoreInit.MetaStoreInitData metaStoreInitData = new MetaStoreInit.MetaStoreInitData(); - private final HiveConf hiveConf; // base configuration - private final Configuration configuration; // active configuration + private final HiveConf origConf; // base configuration + private final Configuration activeConf; // active configuration - private RetryingHMSHandler(HiveConf hiveConf, String name, boolean local) throws MetaException { - this.hiveConf = hiveConf; - this.base = new HiveMetaStore.HMSHandler(name, hiveConf, false); + private RetryingHMSHandler(HiveConf hiveConf, IHMSHandler baseHandler, boolean local) throws MetaException { + this.origConf = hiveConf; + this.baseHandler = baseHandler; if (local) { - base.setConf(hiveConf); // tests expect configuration changes applied directly to metastore + baseHandler.setConf(hiveConf); // tests expect configuration changes applied directly to metastore } - configuration = base.getConf(); + activeConf = baseHandler.getConf(); // This has to be called before initializing the instance of HMSHandler // Using the hook on startup ensures that the hook always has priority // over settings in *.xml. The thread local conf needs to be used because at this point // it has already been initialized using hiveConf. - MetaStoreInit.updateConnectionURL(hiveConf, getConf(), null, metaStoreInitData); + MetaStoreInit.updateConnectionURL(hiveConf, getActiveConf(), null, metaStoreInitData); - base.init(); + baseHandler.init(); } - public static IHMSHandler getProxy(HiveConf hiveConf, String name, boolean local) + public static IHMSHandler getProxy(HiveConf hiveConf, IHMSHandler baseHandler, boolean local) throws MetaException { - RetryingHMSHandler handler = new RetryingHMSHandler(hiveConf, name, local); + RetryingHMSHandler handler = new RetryingHMSHandler(hiveConf, baseHandler, local); return (IHMSHandler) Proxy.newProxyInstance( RetryingHMSHandler.class.getClassLoader(), @@ -79,15 +79,15 @@ public static IHMSHandler getProxy(HiveConf hiveConf, String name, boolean local public Object invoke(final Object proxy, final Method method, final Object[] args) throws Throwable { boolean gotNewConnectUrl = false; - boolean reloadConf = HiveConf.getBoolVar(hiveConf, + boolean reloadConf = HiveConf.getBoolVar(origConf, HiveConf.ConfVars.HMSHANDLERFORCERELOADCONF); - long retryInterval = HiveConf.getTimeVar(hiveConf, + long retryInterval = HiveConf.getTimeVar(origConf, HiveConf.ConfVars.HMSHANDLERINTERVAL, TimeUnit.MILLISECONDS); - int retryLimit = HiveConf.getIntVar(hiveConf, + int retryLimit = HiveConf.getIntVar(origConf, HiveConf.ConfVars.HMSHANDLERATTEMPTS); if (reloadConf) { - MetaStoreInit.updateConnectionURL(hiveConf, getConf(), + MetaStoreInit.updateConnectionURL(origConf, getActiveConf(), null, metaStoreInitData); } @@ -96,9 +96,9 @@ public Object invoke(final Object proxy, final Method method, final Object[] arg while (true) { try { if (reloadConf || gotNewConnectUrl) { - base.setConf(getConf()); + baseHandler.setConf(getActiveConf()); } - return method.invoke(base, args); + return method.invoke(baseHandler, args); } catch (javax.jdo.JDOException e) { caughtException = e; @@ -158,13 +158,13 @@ public Object invoke(final Object proxy, final Method method, final Object[] arg Thread.sleep(retryInterval); // If we have a connection error, the JDO connection URL hook might // provide us with a new URL to access the datastore. - String lastUrl = MetaStoreInit.getConnectionURL(getConf()); - gotNewConnectUrl = MetaStoreInit.updateConnectionURL(hiveConf, getConf(), + String lastUrl = MetaStoreInit.getConnectionURL(getActiveConf()); + gotNewConnectUrl = MetaStoreInit.updateConnectionURL(origConf, getActiveConf(), lastUrl, metaStoreInitData); } } - public Configuration getConf() { - return configuration; + public Configuration getActiveConf() { + return activeConf; } } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java b/metastore/src/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java index cd8c5c2..0f99cf3 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java @@ -417,6 +417,7 @@ public static ColumnStatisticsDesc getPartitionColumnStatisticsDesc( public static void fillColumnStatisticsData(String colType, ColumnStatisticsData data, Object llow, Object lhigh, Object dlow, Object dhigh, Object declow, Object dechigh, Object nulls, Object dist, Object avglen, Object maxlen, Object trues, Object falses) throws MetaException { + colType = colType.toLowerCase(); if (colType.equals("boolean")) { BooleanColumnStatsData boolStats = new BooleanColumnStatsData(); boolStats.setNumFalses(extractSqlLong(falses)); diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddIndexEvent.java b/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddIndexEvent.java new file mode 100644 index 0000000..43ac0aa --- /dev/null +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddIndexEvent.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.events; + +import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hadoop.hive.metastore.api.Index; + +public class AddIndexEvent extends ListenerEvent { + + private final Index index; + + public AddIndexEvent(Index index, boolean status, HiveMetaStore.HMSHandler handler) { + super(status, handler); + this.index = index; + } + + public Index getIndex() { + return index; + } +} diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterIndexEvent.java b/metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterIndexEvent.java new file mode 100644 index 0000000..4a49700 --- /dev/null +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterIndexEvent.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.events; + +import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hadoop.hive.metastore.api.Index; + +public class AlterIndexEvent extends ListenerEvent { + + private final Index newIndex; + private final Index oldIndex; + + public AlterIndexEvent(Index oldIndex, Index newIndex, boolean status, + HiveMetaStore.HMSHandler handler) { + super(status, handler); + this.oldIndex = oldIndex; + this.newIndex = newIndex; + } + + public Index getOldIndex() { + return oldIndex; + } + + public Index getNewIndex() { + return newIndex; + } +} diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/DropIndexEvent.java b/metastore/src/java/org/apache/hadoop/hive/metastore/events/DropIndexEvent.java new file mode 100644 index 0000000..06f2302 --- /dev/null +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/events/DropIndexEvent.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.events; + +import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hadoop.hive.metastore.api.Index; + +public class DropIndexEvent extends ListenerEvent { + + private final Index index; + + public DropIndexEvent(Index index, boolean status, HiveMetaStore.HMSHandler handler) { + super(status, handler); + this.index = index; + } + + public Index getIndex() { + return index; + } +} diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAddIndexEvent.java b/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAddIndexEvent.java new file mode 100644 index 0000000..baa04a5 --- /dev/null +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAddIndexEvent.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.events; + +import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hadoop.hive.metastore.api.Index; + +public class PreAddIndexEvent extends PreEventContext { + +private final Index table; + + public PreAddIndexEvent(Index table, HiveMetaStore.HMSHandler handler) { + super(PreEventType.ADD_INDEX, handler); + this.table = table; + } + + public Index getIndex() { + return table; + } +} + diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAlterIndexEvent.java b/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAlterIndexEvent.java new file mode 100644 index 0000000..97cf305 --- /dev/null +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAlterIndexEvent.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.events; + +import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hadoop.hive.metastore.api.Index; + +public class PreAlterIndexEvent extends PreEventContext { + + private final Index newIndex; + private final Index oldIndex; + + public PreAlterIndexEvent(Index oldIndex, Index newIndex, HiveMetaStore.HMSHandler handler) { + super(PreEventType.ALTER_INDEX, handler); + this.oldIndex = oldIndex; + this.newIndex = newIndex; + } + + public Index getOldIndex() { + return oldIndex; + } + + public Index getNewIndex() { + return newIndex; + } +} + diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropIndexEvent.java b/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropIndexEvent.java new file mode 100644 index 0000000..437e5c1 --- /dev/null +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropIndexEvent.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.events; + +import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hadoop.hive.metastore.api.Index; + +public class PreDropIndexEvent extends PreEventContext { + + private final Index index; + + public PreDropIndexEvent(Index index, HiveMetaStore.HMSHandler handler) { + super(PreEventType.DROP_INDEX, handler); + this.index = index; + } + + public Index getIndex() { + return index; + } +} diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreEventContext.java b/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreEventContext.java index dbc3247..ee24a35 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreEventContext.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreEventContext.java @@ -39,7 +39,10 @@ LOAD_PARTITION_DONE, AUTHORIZATION_API_CALL, READ_TABLE, - READ_DATABASE + READ_DATABASE, + ADD_INDEX, + ALTER_INDEX, + DROP_INDEX } private final PreEventType eventType; diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java index d3aa66f..c4a9581 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java @@ -547,10 +547,27 @@ public void revokeTimedoutWorkers(long timeout) throws MetaException { Statement stmt = null; ResultSet rs = null; try { + String quote = getIdentifierQuoteString(dbConn); stmt = dbConn.createStatement(); - String s = "SELECT COLUMN_NAME FROM " + (ci.partName == null ? "TAB_COL_STATS" : "PART_COL_STATS") + StringBuilder bldr = new StringBuilder(); + bldr.append("SELECT ").append(quote).append("COLUMN_NAME").append(quote) + .append(" FROM ") + .append(quote).append((ci.partName == null ? "TAB_COL_STATS" : "PART_COL_STATS")) + .append(quote) + .append(" WHERE ") + .append(quote).append("DB_NAME").append(quote).append(" = '").append(ci.dbname) + .append("' AND ").append(quote).append("TABLE_NAME").append(quote) + .append(" = '").append(ci.tableName).append("'"); + if (ci.partName != null) { + bldr.append(" AND ").append(quote).append("PARTITION_NAME").append(quote).append(" = '") + .append(ci.partName).append("'"); + } + String s = bldr.toString(); + + /*String s = "SELECT COLUMN_NAME FROM " + (ci.partName == null ? "TAB_COL_STATS" : + "PART_COL_STATS") + " WHERE DB_NAME='" + ci.dbname + "' AND TABLE_NAME='" + ci.tableName + "'" - + (ci.partName == null ? "" : " AND PARTITION_NAME='" + ci.partName + "'"); + + (ci.partName == null ? "" : " AND PARTITION_NAME='" + ci.partName + "'");*/ LOG.debug("Going to execute <" + s + ">"); rs = stmt.executeQuery(s); List columns = new ArrayList(); diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java index 6f44169..567e2e6 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java @@ -90,6 +90,8 @@ // Transaction timeout, in milliseconds. private long timeout; + private String identifierQuoteString; // quotes to use for quoting tables, where necessary + // DEADLOCK DETECTION AND HANDLING // A note to developers of this class. ALWAYS access HIVE_LOCKS before TXNS to avoid deadlock // between simultaneous accesses. ALWAYS access TXN_COMPONENTS before HIVE_LOCKS . @@ -960,6 +962,19 @@ protected long getDbTime(Connection conn) throws MetaException { } } + /** + * Determine the String that should be used to quote identifiers. + * @param conn Active connection + * @return quotes + * @throws SQLException + */ + protected String getIdentifierQuoteString(Connection conn) throws SQLException { + if (identifierQuoteString == null) { + identifierQuoteString = conn.getMetaData().getIdentifierQuoteString(); + } + return identifierQuoteString; + } + protected enum DatabaseProduct { DERBY, MYSQL, POSTGRES, ORACLE, SQLSERVER} /** diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyListener.java b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyListener.java index 37daf52..a3b16d0 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyListener.java +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyListener.java @@ -23,13 +23,16 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.events.AddIndexEvent; import org.apache.hadoop.hive.metastore.events.AddPartitionEvent; +import org.apache.hadoop.hive.metastore.events.AlterIndexEvent; import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent; import org.apache.hadoop.hive.metastore.events.AlterTableEvent; import org.apache.hadoop.hive.metastore.events.ConfigChangeEvent; import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent; import org.apache.hadoop.hive.metastore.events.CreateTableEvent; import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent; +import org.apache.hadoop.hive.metastore.events.DropIndexEvent; import org.apache.hadoop.hive.metastore.events.DropPartitionEvent; import org.apache.hadoop.hive.metastore.events.DropTableEvent; import org.apache.hadoop.hive.metastore.events.ListenerEvent; @@ -60,52 +63,70 @@ public DummyListener(Configuration config) { @Override public void onConfigChange(ConfigChangeEvent configChange) { - notifyList.add(configChange); + addEvent(configChange); } @Override public void onAddPartition(AddPartitionEvent partition) throws MetaException { - notifyList.add(partition); + addEvent(partition); } @Override public void onCreateDatabase(CreateDatabaseEvent db) throws MetaException { - notifyList.add(db); + addEvent(db); } @Override public void onCreateTable(CreateTableEvent table) throws MetaException { - notifyList.add(table); + addEvent(table); } @Override public void onDropDatabase(DropDatabaseEvent db) throws MetaException { - notifyList.add(db); + addEvent(db); } @Override public void onDropPartition(DropPartitionEvent partition) throws MetaException { - notifyList.add(partition); + addEvent(partition); } @Override public void onDropTable(DropTableEvent table) throws MetaException { - notifyList.add(table); + addEvent(table); } @Override public void onAlterTable(AlterTableEvent event) throws MetaException { - notifyList.add(event); + addEvent(event); } @Override public void onAlterPartition(AlterPartitionEvent event) throws MetaException { - notifyList.add(event); + addEvent(event); } @Override public void onLoadPartitionDone(LoadPartitionDoneEvent partEvent) throws MetaException { - notifyList.add(partEvent); + addEvent(partEvent); } + @Override + public void onAddIndex(AddIndexEvent indexEvent) throws MetaException { + addEvent(indexEvent); + } + + @Override + public void onDropIndex(DropIndexEvent indexEvent) throws MetaException { + addEvent(indexEvent); + } + + @Override + public void onAlterIndex(AlterIndexEvent indexEvent) throws MetaException { + addEvent(indexEvent); + } + + private void addEvent(ListenerEvent event) { + notifyList.add(event); + } } diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyPreListener.java b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyPreListener.java index 9a943b2..7ff6f92 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyPreListener.java +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyPreListener.java @@ -30,7 +30,7 @@ * * DummyPreListener. * - * An implemenation of MetaStorePreEventListener which stores the Events it's seen in a list. + * An implementation of MetaStorePreEventListener which stores the Events it's seen in a list. */ public class DummyPreListener extends MetaStorePreEventListener { diff --git a/odbc/pom.xml b/odbc/pom.xml index e233533..e7710fa 100644 --- a/odbc/pom.xml +++ b/odbc/pom.xml @@ -19,7 +19,7 @@ org.apache.hive hive - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../pom.xml diff --git a/packaging/pom.xml b/packaging/pom.xml index ebf34ed..1b4ea70 100644 --- a/packaging/pom.xml +++ b/packaging/pom.xml @@ -19,7 +19,7 @@ org.apache.hive hive - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index 27ebfa5..92cefd2 100644 --- a/pom.xml +++ b/pom.xml @@ -21,7 +21,7 @@ org.apache.hive hive - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT pom Hive @@ -53,7 +53,7 @@ - 0.14.0 + 0.15.0 UTF-8 @@ -133,7 +133,7 @@ 1.1 3.5.2 20090211 - 4.10 + 4.11 2.22 0.9.0 0.9.0 @@ -152,7 +152,7 @@ 1.0.1 1.7.5 4.0.4 - 0.5.1 + 0.5.2-SNAPSHOT 2.2.0 1.2.0-SNAPSHOT 2.10 @@ -165,6 +165,7 @@ 3.4.6 1.1 2.4.0 + 2.6.0 @@ -486,7 +487,13 @@ - + + org.apache.curator + curator-framework + ${curator.version} + + + org.codehaus.groovy groovy-all ${groovy.version} diff --git a/ql/pom.xml b/ql/pom.xml index c796c95..83df0ac 100644 --- a/ql/pom.xml +++ b/ql/pom.xml @@ -19,7 +19,7 @@ org.apache.hive hive - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../pom.xml @@ -28,7 +28,7 @@ Hive Query Language - 0.9.1-incubating-SNAPSHOT + 0.9.2-incubating-SNAPSHOT .. @@ -183,9 +183,9 @@ ${datanucleus-core.version} - org.apache.optiq - optiq-core - ${optiq.version} + org.apache.calcite + calcite-core + ${calcite.version} {operator --> context} + private final Map, MapOpCtx>> opCtxMap = + new HashMap, MapOpCtx>>(); + // child operator --> object inspector (converted OI if it's needed) + private final Map, StructObjectInspector> childrenOpToOI = + new HashMap, StructObjectInspector>(); - return false; - } - - @Override - public int hashCode() { - int ret = (path == null) ? 0 : path.hashCode(); - ret += (alias == null) ? 0 : alias.hashCode(); - ret += (op == null) ? 0 : op.hashCode(); - return ret; - } - } + // context for current input file + protected transient MapOpCtx[] currentCtxs; + private transient final Map normalizedPaths = new HashMap(); protected static class MapOpCtx { - StructObjectInspector tblRawRowObjectInspector; // columns + final String alias; + final Operator op; + final PartitionDesc partDesc; + StructObjectInspector partObjectInspector; // partition columns StructObjectInspector vcsObjectInspector; // virtual columns StructObjectInspector rowObjectInspector; @@ -144,6 +118,12 @@ public int hashCode() { List vcs; Object[] vcValues; + public MapOpCtx(String alias, Operator op, PartitionDesc partDesc) { + this.alias = alias; + this.op = op; + this.partDesc = partDesc; + } + private boolean isPartitioned() { return partObjectInspector != null; } @@ -152,12 +132,30 @@ private boolean hasVC() { return vcsObjectInspector != null; } - private Object readRow(Writable value) throws SerDeException { - return partTblObjectInspectorConverter.convert(deserializer.deserialize(value)); + private Object readRow(Writable value, ExecMapperContext context) throws SerDeException { + Object deserialized = deserializer.deserialize(value); + Object row = partTblObjectInspectorConverter.convert(deserialized); + if (hasVC()) { + rowWithPartAndVC[0] = row; + if (context != null) { + populateVirtualColumnValues(context, vcs, vcValues, deserializer); + } + int vcPos = isPartitioned() ? 2 : 1; + rowWithPartAndVC[vcPos] = vcValues; + return rowWithPartAndVC; + } else if (isPartitioned()) { + rowWithPart[0] = row; + return rowWithPart; + } + return row; } - public StructObjectInspector getRowObjectInspector() { - return rowObjectInspector; + public boolean forward(Object row) throws HiveException { + if (op.getDone()) { + return false; + } + op.processOp(row, 0); + return true; } } @@ -170,20 +168,19 @@ public StructObjectInspector getRowObjectInspector() { * @param mapWork * @throws HiveException */ - public void initializeAsRoot(Configuration hconf, MapWork mapWork) - throws HiveException { + @VisibleForTesting + void initializeAsRoot(JobConf hconf, MapWork mapWork) throws Exception { setConf(mapWork); setChildren(hconf); + setExecContext(new ExecMapperContext(hconf)); initialize(hconf, null); } - private MapOpCtx initObjectInspector(Configuration hconf, MapInputPath ctx, - Map convertedOI) throws Exception { - - PartitionDesc pd = ctx.partDesc; + private MapOpCtx initObjectInspector(Configuration hconf, MapOpCtx opCtx, + StructObjectInspector tableRowOI) throws Exception { + PartitionDesc pd = opCtx.partDesc; TableDesc td = pd.getTableDesc(); - MapOpCtx opCtx = new MapOpCtx(); // Use table properties in case of unpartitioned tables, // and the union of table properties and partition properties, with partition // taking precedence, in the case of partitioned tables @@ -194,18 +191,13 @@ private MapOpCtx initObjectInspector(Configuration hconf, MapInputPath ctx, opCtx.tableName = String.valueOf(overlayedProps.getProperty("name")); opCtx.partName = String.valueOf(partSpec); - - Class serdeclass = hconf.getClassByName(pd.getSerdeClassName()); - opCtx.deserializer = (Deserializer) serdeclass.newInstance(); - SerDeUtils.initializeSerDe(opCtx.deserializer, hconf, td.getProperties(), pd.getProperties()); + opCtx.deserializer = pd.getDeserializer(hconf); StructObjectInspector partRawRowObjectInspector = (StructObjectInspector) opCtx.deserializer.getObjectInspector(); - opCtx.tblRawRowObjectInspector = convertedOI.get(td); - - opCtx.partTblObjectInspectorConverter = ObjectInspectorConverters.getConverter( - partRawRowObjectInspector, opCtx.tblRawRowObjectInspector); + opCtx.partTblObjectInspectorConverter = + ObjectInspectorConverters.getConverter(partRawRowObjectInspector, tableRowOI); // Next check if this table has partitions and if so // get the list of partition names as well as allocate @@ -253,8 +245,8 @@ private MapOpCtx initObjectInspector(Configuration hconf, MapInputPath ctx, // The op may not be a TableScan for mapjoins // Consider the query: select /*+MAPJOIN(a)*/ count(*) FROM T1 a JOIN T2 b ON a.key = b.key; // In that case, it will be a Select, but the rowOI need not be amended - if (ctx.op instanceof TableScanOperator) { - TableScanOperator tsOp = (TableScanOperator) ctx.op; + if (opCtx.op instanceof TableScanOperator) { + TableScanOperator tsOp = (TableScanOperator) opCtx.op; TableScanDesc tsDesc = tsOp.getConf(); if (tsDesc != null && tsDesc.hasVirtualCols()) { opCtx.vcs = tsDesc.getVirtualCols(); @@ -268,11 +260,11 @@ private MapOpCtx initObjectInspector(Configuration hconf, MapInputPath ctx, } } if (!opCtx.hasVC() && !opCtx.isPartitioned()) { - opCtx.rowObjectInspector = opCtx.tblRawRowObjectInspector; + opCtx.rowObjectInspector = tableRowOI; return opCtx; } List inspectors = new ArrayList(); - inspectors.add(opCtx.tblRawRowObjectInspector); + inspectors.add(tableRowOI); if (opCtx.isPartitioned()) { inspectors.add(opCtx.partObjectInspector); } @@ -302,19 +294,14 @@ private MapOpCtx initObjectInspector(Configuration hconf, MapInputPath ctx, for (String onefile : conf.getPathToAliases().keySet()) { PartitionDesc pd = conf.getPathToPartitionInfo().get(onefile); TableDesc tableDesc = pd.getTableDesc(); - Properties tblProps = tableDesc.getProperties(); - Class sdclass = hconf.getClassByName(pd.getSerdeClassName()); - Deserializer partDeserializer = (Deserializer) sdclass.newInstance(); - SerDeUtils.initializeSerDe(partDeserializer, hconf, tblProps, pd.getProperties()); - StructObjectInspector partRawRowObjectInspector = (StructObjectInspector) partDeserializer - .getObjectInspector(); + Deserializer partDeserializer = pd.getDeserializer(hconf); + StructObjectInspector partRawRowObjectInspector = + (StructObjectInspector) partDeserializer.getObjectInspector(); StructObjectInspector tblRawRowObjectInspector = tableDescOI.get(tableDesc); if ((tblRawRowObjectInspector == null) || (identityConverterTableDesc.contains(tableDesc))) { - sdclass = hconf.getClassByName(tableDesc.getSerdeClassName()); - Deserializer tblDeserializer = (Deserializer) sdclass.newInstance(); - SerDeUtils.initializeSerDe(tblDeserializer, hconf, tblProps, null); + Deserializer tblDeserializer = tableDesc.getDeserializer(hconf); tblRawRowObjectInspector = (StructObjectInspector) ObjectInspectorConverters.getConvertedOI( partRawRowObjectInspector, @@ -338,155 +325,153 @@ else if (partRawRowObjectInspector.equals(tblRawRowObjectInspector)) { return tableDescOI; } - public void setChildren(Configuration hconf) throws HiveException { - Path fpath = IOContext.get(hconf).getInputPath(); - - boolean schemeless = fpath.toUri().getScheme() == null; + public void setChildren(Configuration hconf) throws Exception { List> children = new ArrayList>(); Map convertedOI = getConvertedOI(hconf); - try { - for (Map.Entry> entry : conf.getPathToAliases().entrySet()) { - String onefile = entry.getKey(); - List aliases = entry.getValue(); + for (Map.Entry> entry : conf.getPathToAliases().entrySet()) { + String onefile = entry.getKey(); + List aliases = entry.getValue(); + PartitionDesc partDesc = conf.getPathToPartitionInfo().get(onefile); - Path onepath = new Path(onefile); - if (schemeless) { - onepath = new Path(onepath.toUri().getPath()); + for (String alias : aliases) { + Operator op = conf.getAliasToWork().get(alias); + if (isLogDebugEnabled) { + LOG.debug("Adding alias " + alias + " to work list for file " + + onefile); } + Map, MapOpCtx> contexts = opCtxMap.get(onefile); + if (contexts == null) { + opCtxMap.put(onefile, contexts = new LinkedHashMap, MapOpCtx>()); + } + if (contexts.containsKey(op)) { + continue; + } + MapOpCtx context = new MapOpCtx(alias, op, partDesc); + StructObjectInspector tableRowOI = convertedOI.get(partDesc.getTableDesc()); + contexts.put(op, initObjectInspector(hconf, context, tableRowOI)); - PartitionDesc partDesc = conf.getPathToPartitionInfo().get(onefile); - - for (String onealias : aliases) { - Operator op = conf.getAliasToWork().get(onealias); - if (LOG.isDebugEnabled()) { - LOG.debug("Adding alias " + onealias + " to work list for file " - + onefile); - } - MapInputPath inp = new MapInputPath(onefile, onealias, op, partDesc); - if (opCtxMap.containsKey(inp)) { - continue; - } - MapOpCtx opCtx = initObjectInspector(hconf, inp, convertedOI); - opCtxMap.put(inp, opCtx); - - op.setParentOperators(new ArrayList>()); + if (children.contains(op) == false) { + op.setParentOperators(new ArrayList>(1)); op.getParentOperators().add(this); - // check for the operators who will process rows coming to this Map - // Operator - if (!onepath.toUri().relativize(fpath.toUri()).equals(fpath.toUri())) { - children.add(op); - childrenOpToOpCtxMap.put(op, opCtx); - LOG.info("dump " + op + " " - + opCtxMap.get(inp).rowObjectInspector.getTypeName()); - } - current = opCtx; // just need for TestOperators.testMapOperator + children.add(op); } } + } - if (children.size() == 0) { - // didn't find match for input file path in configuration! - // serious problem .. - LOG.error("Configuration does not have any alias for path: " - + fpath.toUri()); - throw new HiveException("Configuration and input path are inconsistent"); + initOperatorContext(children); + + // we found all the operators that we are supposed to process. + setChildOperators(children); + } + + private void initOperatorContext(List> children) + throws HiveException { + for (Map, MapOpCtx> contexts : opCtxMap.values()) { + for (MapOpCtx context : contexts.values()) { + if (!children.contains(context.op)) { + continue; + } + StructObjectInspector prev = + childrenOpToOI.put(context.op, context.rowObjectInspector); + if (prev != null && !prev.equals(context.rowObjectInspector)) { + throw new HiveException("Conflict on row inspector for " + context.alias); + } + if (isLogDebugEnabled) { + LOG.debug("dump " + context.op + " " + context.rowObjectInspector.getTypeName()); + } } + } + } - // we found all the operators that we are supposed to process. - setChildOperators(children); - } catch (Exception e) { - throw new HiveException(e); + private String getNominalPath(Path fpath) { + String nominal = null; + boolean schemaless = fpath.toUri().getScheme() == null; + for (String onefile : conf.getPathToAliases().keySet()) { + Path onepath = normalizePath(onefile, schemaless); + // check for the operators who will process rows coming to this Map Operator + if (onepath.toUri().relativize(fpath.toUri()).equals(fpath.toUri())) { + // not from this + continue; + } + if (nominal != null) { + throw new IllegalStateException("Ambiguous input path " + fpath); + } + nominal = onefile; } + if (nominal == null) { + throw new IllegalStateException("Invalid input path " + fpath); + } + return nominal; } @Override public void initializeOp(Configuration hconf) throws HiveException { // set that parent initialization is done and call initialize on children state = State.INIT; - statsMap.put(Counter.DESERIALIZE_ERRORS, deserialize_error_count); - - List> children = getChildOperators(); - - for (Entry, MapOpCtx> entry : childrenOpToOpCtxMap - .entrySet()) { - Operator child = entry.getKey(); - MapOpCtx mapOpCtx = entry.getValue(); - // Add alias, table name, and partitions to hadoop conf so that their - // children will inherit these - HiveConf.setVar(hconf, HiveConf.ConfVars.HIVETABLENAME, mapOpCtx.tableName); - HiveConf.setVar(hconf, HiveConf.ConfVars.HIVEPARTITIONNAME, mapOpCtx.partName); - child.initialize(hconf, new ObjectInspector[] {mapOpCtx.rowObjectInspector}); + statsMap.put(Counter.DESERIALIZE_ERRORS.toString(), deserialize_error_count); + + numRows = 0; + + String context = hconf.get(Operator.CONTEXT_NAME_KEY, ""); + if (context != null && !context.isEmpty()) { + context = "_" + context.replace(" ","_"); } + statsMap.put(Counter.RECORDS_IN + context, recordCounter); - for (Entry entry : opCtxMap.entrySet()) { - MapInputPath input = entry.getKey(); - MapOpCtx mapOpCtx = entry.getValue(); - // Add alias, table name, and partitions to hadoop conf so that their - // children will inherit these - HiveConf.setVar(hconf, HiveConf.ConfVars.HIVETABLENAME, mapOpCtx.tableName); - HiveConf.setVar(hconf, HiveConf.ConfVars.HIVEPARTITIONNAME, mapOpCtx.partName); - - Operator op = input.op; - if (children.indexOf(op) == -1) { - // op is not in the children list, so need to remember it and close it afterwards - if (extraChildrenToClose == null) { - extraChildrenToClose = new ArrayList>(); - } - extraChildrenToClose.add(op); - op.initialize(hconf, new ObjectInspector[] {entry.getValue().rowObjectInspector}); - } + for (Entry, StructObjectInspector> entry : childrenOpToOI.entrySet()) { + Operator child = entry.getKey(); + child.initialize(hconf, new ObjectInspector[] {entry.getValue()}); } } - /** - * close extra child operators that are initialized but are not executed. - */ @Override public void closeOp(boolean abort) throws HiveException { - if (extraChildrenToClose != null) { - for (Operator op : extraChildrenToClose) { - op.close(abort); - } - } + recordCounter.set(numRows); + super.closeOp(abort); } // Find context for current input file @Override public void cleanUpInputFileChangedOp() throws HiveException { + super.cleanUpInputFileChangedOp(); Path fpath = getExecContext().getCurrentInputPath(); - - for (String onefile : conf.getPathToAliases().keySet()) { - Path onepath = normalizePath(onefile); - // check for the operators who will process rows coming to this Map - // Operator - if (onepath.toUri().relativize(fpath.toUri()).equals(fpath.toUri())) { - // not from this - continue; - } - PartitionDesc partDesc = conf.getPathToPartitionInfo().get(onefile); - for (String onealias : conf.getPathToAliases().get(onefile)) { - Operator op = conf.getAliasToWork().get(onealias); - MapInputPath inp = new MapInputPath(onefile, onealias, op, partDesc); - MapOpCtx context = opCtxMap.get(inp); - if (context != null) { - current = context; - LOG.info("Processing alias " + onealias + " for file " + onefile); - return; + String nominalPath = getNominalPath(fpath); + Map, MapOpCtx> contexts = opCtxMap.get(nominalPath); + if (isLogInfoEnabled) { + StringBuilder builder = new StringBuilder(); + for (MapOpCtx context : contexts.values()) { + if (builder.length() > 0) { + builder.append(", "); } + builder.append(context.alias); + } + if (isLogDebugEnabled) { + LOG.info("Processing alias(es) " + builder.toString() + " for file " + fpath); } } - throw new IllegalStateException("Invalid path " + fpath); + // Add alias, table name, and partitions to hadoop conf so that their + // children will inherit these + for (Entry, MapOpCtx> entry : contexts.entrySet()) { + Operator operator = entry.getKey(); + MapOpCtx context = entry.getValue(); + operator.setInputContext(nominalPath, context.tableName, context.partName); + } + currentCtxs = contexts.values().toArray(new MapOpCtx[contexts.size()]); } - private Path normalizePath(String onefile) { + private Path normalizePath(String onefile, boolean schemaless) { //creating Path is expensive, so cache the corresponding //Path object in normalizedPaths Path path = normalizedPaths.get(onefile); - if(path == null){ + if (path == null) { path = new Path(onefile); + if (schemaless && path.toUri().getScheme() != null) { + path = new Path(path.toUri().getPath()); + } normalizedPaths.put(onefile, path); } return path; @@ -500,50 +485,48 @@ public void process(Writable value) throws HiveException { // The child operators cleanup if input file has changed cleanUpInputFileChanged(); } - Object row; - try { - row = current.readRow(value); - if (current.hasVC()) { - current.rowWithPartAndVC[0] = row; - if (context != null) { - populateVirtualColumnValues(context, current.vcs, current.vcValues, current.deserializer); - } - int vcPos = current.isPartitioned() ? 2 : 1; - current.rowWithPartAndVC[vcPos] = current.vcValues; - row = current.rowWithPartAndVC; - } else if (current.isPartitioned()) { - current.rowWithPart[0] = row; - row = current.rowWithPart; - } - } catch (Exception e) { - // Serialize the row and output. - String rawRowString; + int childrenDone = 0; + for (MapOpCtx current : currentCtxs) { + Object row = null; try { - rawRowString = value.toString(); - } catch (Exception e2) { - rawRowString = "[Error getting row data with exception " + - StringUtils.stringifyException(e2) + " ]"; + row = current.readRow(value, context); + if (!current.forward(row)) { + childrenDone++; + } + } catch (Exception e) { + // TODO: policy on deserialization errors + String message = toErrorMessage(value, row, current.rowObjectInspector); + if (row == null) { + deserialize_error_count.set(deserialize_error_count.get() + 1); + throw new HiveException("Hive Runtime Error while processing writable " + message, e); + } + throw new HiveException("Hive Runtime Error while processing row " + message, e); } + } + rowsForwarded(childrenDone, 1); + } - // TODO: policy on deserialization errors - deserialize_error_count.set(deserialize_error_count.get() + 1); - throw new HiveException("Hive Runtime Error while processing writable " + rawRowString, e); + protected final void rowsForwarded(int childrenDone, int rows) { + numRows += rows; + if (isLogInfoEnabled) { + while (numRows >= cntr) { + cntr *= 10; + LOG.info(toString() + ": records read - " + numRows); + } } + if (childrenDone == currentCtxs.length) { + setDone(true); + } + } - // The row has been converted to comply with table schema, irrespective of partition schema. - // So, use tblOI (and not partOI) for forwarding + private String toErrorMessage(Writable value, Object row, ObjectInspector inspector) { try { - forward(row, current.rowObjectInspector); - } catch (Exception e) { - // Serialize the row and output the error message. - String rowString; - try { - rowString = SerDeUtils.getJSONString(row, current.rowObjectInspector); - } catch (Exception e2) { - rowString = "[Error getting row data with exception " + - StringUtils.stringifyException(e2) + " ]"; + if (row != null) { + return SerDeUtils.getJSONString(row, inspector); } - throw new HiveException("Hive Runtime Error while processing row " + rowString, e); + return String.valueOf(value); + } catch (Exception e) { + return "[Error getting row data with exception " + StringUtils.stringifyException(e) + " ]"; } } @@ -639,4 +622,16 @@ public OperatorType getType() { public Map getTagToOperatorTree() { return MapRecordProcessor.getConnectOps(); } + + public void initializeContexts() { + Path fpath = getExecContext().getCurrentInputPath(); + String nominalPath = getNominalPath(fpath); + Map, MapOpCtx> contexts = opCtxMap.get(nominalPath); + currentCtxs = contexts.values().toArray(new MapOpCtx[contexts.size()]); + } + + public Deserializer getCurrentDeserializer() { + + return currentCtxs[0].deserializer; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java index 3dc7c76..273691e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java @@ -61,6 +61,7 @@ public static final String HIVECOUNTERCREATEDFILES = "CREATED_FILES"; public static final String HIVECOUNTERFATAL = "FATAL_ERROR"; + public static final String CONTEXT_NAME_KEY = "__hive.context.name"; private transient Configuration configuration; protected List> childOperators; @@ -210,11 +211,14 @@ public RowSchema getSchema() { // non-bean .. - protected transient HashMap, LongWritable> statsMap = new HashMap, LongWritable>(); + protected transient Map statsMap = new HashMap(); @SuppressWarnings("rawtypes") protected transient OutputCollector out; - protected transient Log LOG = LogFactory.getLog(this.getClass().getName()); - protected transient boolean isLogInfoEnabled = LOG.isInfoEnabled(); + protected transient final Log LOG = LogFactory.getLog(getClass().getName()); + protected transient final Log PLOG = LogFactory.getLog(Operator.class.getName()); // for simple disabling logs from all operators + protected transient final boolean isLogInfoEnabled = LOG.isInfoEnabled() && PLOG.isInfoEnabled(); + protected transient final boolean isLogDebugEnabled = LOG.isDebugEnabled() && PLOG.isDebugEnabled(); + protected transient final boolean isLogTraceEnabled = LOG.isTraceEnabled() && PLOG.isTraceEnabled(); protected transient String alias; protected transient Reporter reporter; protected transient String id; @@ -287,9 +291,9 @@ public void setAlias(String alias) { } } - public Map, Long> getStats() { - HashMap, Long> ret = new HashMap, Long>(); - for (Enum one : statsMap.keySet()) { + public Map getStats() { + HashMap ret = new HashMap(); + for (String one : statsMap.keySet()) { ret.put(one, Long.valueOf(statsMap.get(one).get())); } return (ret); @@ -490,33 +494,45 @@ public ObjectInspector getOutputObjInspector() { public abstract void processOp(Object row, int tag) throws HiveException; protected final void defaultStartGroup() throws HiveException { - LOG.debug("Starting group"); + if (isLogDebugEnabled) { + LOG.debug("Starting group"); + } if (childOperators == null) { return; } - LOG.debug("Starting group for children:"); + if (isLogDebugEnabled) { + LOG.debug("Starting group for children:"); + } for (Operator op : childOperators) { op.startGroup(); } - LOG.debug("Start group Done"); + if (isLogDebugEnabled) { + LOG.debug("Start group Done"); + } } protected final void defaultEndGroup() throws HiveException { - LOG.debug("Ending group"); + if (isLogDebugEnabled) { + LOG.debug("Ending group"); + } if (childOperators == null) { return; } - LOG.debug("Ending group for children:"); + if (isLogDebugEnabled) { + LOG.debug("Ending group for children:"); + } for (Operator op : childOperators) { op.endGroup(); } - LOG.debug("End group Done"); + if (isLogDebugEnabled) { + LOG.debug("End group Done"); + } } // If a operator wants to do some work at the beginning of a group @@ -807,7 +823,7 @@ protected void forward(Object row, ObjectInspector rowInspector) } public void resetStats() { - for (Enum e : statsMap.keySet()) { + for (String e : statsMap.keySet()) { statsMap.get(e).set(0L); } } @@ -840,7 +856,7 @@ public void preorderMap(OperatorFunc opFunc) { } public void logStats() { - for (Enum e : statsMap.keySet()) { + for (String e : statsMap.keySet()) { LOG.info(e.toString() + ":" + statsMap.get(e).toString()); } } @@ -1046,6 +1062,17 @@ public void cleanUpInputFileChanged() throws HiveException { public void cleanUpInputFileChangedOp() throws HiveException { } + // called by map operator. propagated recursively to single parented descendants + public void setInputContext(String inputPath, String tableName, String partitionName) { + if (childOperators != null) { + for (Operator child : childOperators) { + if (child.getNumParent() == 1) { + child.setInputContext(inputPath, tableName, partitionName); + } + } + } + } + public boolean supportSkewJoinOptimization() { return false; } @@ -1263,7 +1290,7 @@ public OpTraits getOpTraits() { } public void setOpTraits(OpTraits metaInfo) { - if (LOG.isDebugEnabled()) { + if (isLogDebugEnabled) { LOG.debug("Setting traits ("+metaInfo+") on "+this); } if (conf != null) { @@ -1274,7 +1301,7 @@ public void setOpTraits(OpTraits metaInfo) { } public void setStatistics(Statistics stats) { - if (LOG.isDebugEnabled()) { + if (isLogDebugEnabled) { LOG.debug("Setting stats ("+stats+") on "+this); } if (conf != null) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java index 2bd40fa..4d9d1da 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java @@ -143,4 +143,8 @@ private static void iterateParents(Operator operator, Function> f } } } + + public static boolean sameRowSchema(Operator operator1, Operator operator2) { + return operator1.getSchema().equals(operator2.getSchema()); + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java index 0fb1c57..5ee8e32 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java @@ -64,6 +64,7 @@ public void processOp(Object row, int tag) throws HiveException { private void processKeyValuePairs(Object key, Object value) throws HiveException { + String filePath = ""; try { OrcFileValueWrapper v; OrcFileKeyWrapper k; @@ -72,6 +73,7 @@ private void processKeyValuePairs(Object key, Object value) } else { k = (OrcFileKeyWrapper) key; } + filePath = k.getInputPath().toUri().getPath(); fixTmpPath(k.getInputPath().getParent()); @@ -131,6 +133,16 @@ private void processKeyValuePairs(Object key, Object value) this.exception = true; closeOp(true); throw new HiveException(e); + } finally { + if (fdis != null) { + try { + fdis.close(); + } catch (IOException e) { + throw new HiveException(String.format("Unable to close file %s", filePath), e); + } finally { + fdis = null; + } + } } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java index 4b267bb..2e6a880 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java @@ -337,17 +337,20 @@ void finishPartition() throws HiveException { handleOutputRows(tabFn.finishPartition()); } else { if ( tabFn.canIterateOutput() ) { - outputPartRowsItr = tabFn.iterator(inputPart.iterator()); + outputPartRowsItr = inputPart == null ? null : + tabFn.iterator(inputPart.iterator()); } else { - outputPart = tabFn.execute(inputPart); - outputPartRowsItr = outputPart.iterator(); + outputPart = inputPart == null ? null : tabFn.execute(inputPart); + outputPartRowsItr = outputPart == null ? null : outputPart.iterator(); } if ( next != null ) { if (!next.isStreaming() && !isOutputIterator() ) { next.inputPart = outputPart; } else { - while(outputPartRowsItr.hasNext() ) { - next.processRow(outputPartRowsItr.next()); + if ( outputPartRowsItr != null ) { + while(outputPartRowsItr.hasNext() ) { + next.processRow(outputPartRowsItr.next()); + } } } } @@ -357,8 +360,10 @@ void finishPartition() throws HiveException { next.finishPartition(); } else { if (!isStreaming() ) { - while(outputPartRowsItr.hasNext() ) { - forward(outputPartRowsItr.next(), outputObjInspector); + if ( outputPartRowsItr != null ) { + while(outputPartRowsItr.hasNext() ) { + forward(outputPartRowsItr.next(), outputObjInspector); + } } } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java index d8698da..bc91529 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java @@ -25,8 +25,6 @@ import java.util.List; import java.util.Random; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.io.AcidUtils; @@ -51,10 +49,13 @@ import org.apache.hadoop.io.BinaryComparable; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.util.hash.MurmurHash; +import static org.apache.hadoop.hive.ql.plan.ReduceSinkDesc.ReducerTraits.UNIFORM; + /** * Reduce Sink Operator sends output to the reduce stage. **/ @@ -65,10 +66,13 @@ PTFUtils.makeTransient(ReduceSinkOperator.class, "inputAliases", "valueIndex"); } - private static final Log LOG = LogFactory.getLog(ReduceSinkOperator.class.getName()); - private static final boolean isInfoEnabled = LOG.isInfoEnabled(); - private static final boolean isDebugEnabled = LOG.isDebugEnabled(); - private static final boolean isTraceEnabled = LOG.isTraceEnabled(); + /** + * Counters. + */ + public static enum Counter { + RECORDS_OUT_INTERMEDIATE + } + private static final long serialVersionUID = 1L; private static final MurmurHash hash = (MurmurHash) MurmurHash.getInstance(); @@ -110,7 +114,7 @@ protected transient int numDistributionKeys; protected transient int numDistinctExprs; protected transient String[] inputAliases; // input aliases of this RS for join (used for PPD) - protected transient boolean autoParallel = false; + protected transient boolean useUniformHash = false; // picks topN K:V pairs from input. protected transient TopNHash reducerHash = new TopNHash(); protected transient HiveKey keyWritable = new HiveKey(); @@ -144,12 +148,25 @@ private StructObjectInspector recIdInspector; // OI for the record identifier private IntObjectInspector bucketInspector; // OI for the bucket field in the record id + protected transient long numRows = 0; + protected transient long cntr = 1; + private final transient LongWritable recordCounter = new LongWritable(); + @Override protected void initializeOp(Configuration hconf) throws HiveException { try { + + numRows = 0; + + String context = hconf.get(Operator.CONTEXT_NAME_KEY, ""); + if (context != null && !context.isEmpty()) { + context = "_" + context.replace(" ","_"); + } + statsMap.put(Counter.RECORDS_OUT_INTERMEDIATE + context, recordCounter); + List keys = conf.getKeyCols(); - if (isDebugEnabled) { + if (isLogDebugEnabled) { LOG.debug("keys size is " + keys.size()); for (ExprNodeDesc k : keys) { LOG.debug("Key exprNodeDesc " + k.getExprString()); @@ -194,7 +211,7 @@ protected void initializeOp(Configuration hconf) throws HiveException { tag = conf.getTag(); tagByte[0] = (byte) tag; skipTag = conf.getSkipTag(); - if (isInfoEnabled) { + if (isLogInfoEnabled) { LOG.info("Using tag = " + tag); } @@ -217,7 +234,7 @@ protected void initializeOp(Configuration hconf) throws HiveException { reducerHash.initialize(limit, memUsage, conf.isMapGroupBy(), this); } - autoParallel = conf.isAutoParallel(); + useUniformHash = conf.getReducerTraits().contains(UNIFORM); firstRow = true; initializeChildren(hconf); @@ -296,7 +313,7 @@ public void processOp(Object row, int tag) throws HiveException { bucketInspector = (IntObjectInspector)bucketField.getFieldObjectInspector(); } - if (isInfoEnabled) { + if (isLogInfoEnabled) { LOG.info("keys are " + conf.getOutputKeyColumnNames() + " num distributions: " + conf.getNumDistributionKeys()); } @@ -339,10 +356,10 @@ public void processOp(Object row, int tag) throws HiveException { final int hashCode; // distKeyLength doesn't include tag, but includes buckNum in cachedKeys[0] - if (autoParallel && partitionEval.length > 0) { + if (useUniformHash && partitionEval.length > 0) { hashCode = computeMurmurHash(firstKey); } else { - hashCode = computeHashCode(row); + hashCode = computeHashCode(row, bucketNumber); } firstKey.setHashCode(hashCode); @@ -391,7 +408,7 @@ private int computeBucketNumber(Object row, int numBuckets) throws HiveException // column directly. Object recIdValue = acidRowInspector.getStructFieldData(row, recIdField); buckNum = bucketInspector.get(recIdInspector.getStructFieldData(recIdValue, bucketField)); - if (isTraceEnabled) { + if (isLogTraceEnabled) { LOG.trace("Acid choosing bucket number " + buckNum); } } else { @@ -438,7 +455,7 @@ protected final int computeMurmurHash(HiveKey firstKey) { return hash.hash(firstKey.getBytes(), firstKey.getDistKeyLength(), 0); } - private int computeHashCode(Object row) throws HiveException { + private int computeHashCode(Object row, int buckNum) throws HiveException { // Evaluate the HashCode int keyHashCode = 0; if (partitionEval.length == 0) { @@ -462,10 +479,11 @@ private int computeHashCode(Object row) throws HiveException { + ObjectInspectorUtils.hashCode(o, partitionObjectInspectors[i]); } } - if (isTraceEnabled) { - LOG.trace("Going to return hash code " + (keyHashCode * 31 + bucketNumber)); + int hashCode = buckNum < 0 ? keyHashCode : keyHashCode * 31 + buckNum; + if (isLogTraceEnabled) { + LOG.trace("Going to return hash code " + hashCode); } - return bucketNumber < 0 ? keyHashCode : keyHashCode * 31 + bucketNumber; + return hashCode; } private boolean partitionKeysAreNull(Object row) throws HiveException { @@ -506,6 +524,13 @@ protected void collect(BytesWritable keyWritable, Writable valueWritable) throws // Since this is a terminal operator, update counters explicitly - // forward is not called if (null != out) { + numRows++; + if (isLogInfoEnabled) { + if (numRows == cntr) { + cntr *= 10; + LOG.info(toString() + ": records written - " + numRows); + } + } out.collect(keyWritable, valueWritable); } } @@ -535,6 +560,10 @@ protected void closeOp(boolean abort) throws HiveException { } super.closeOp(abort); out = null; + if (isLogInfoEnabled) { + LOG.info(toString() + ": records written - " + numRows); + } + recordCounter.set(numRows); } /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/RowSchema.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/RowSchema.java index 71cc7eb..450d7f3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/RowSchema.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/RowSchema.java @@ -20,6 +20,7 @@ import java.io.Serializable; import java.util.ArrayList; +import java.util.Iterator; /** * RowSchema Implementation. @@ -49,6 +50,51 @@ public void setSignature(ArrayList signature) { } @Override + public boolean equals(Object obj) { + if (!(obj instanceof RowSchema) || (obj == null)) { + return false; + } + if(this == obj) { + return true; + } + + RowSchema dest = (RowSchema)obj; + if(this.signature == null && dest.getSignature() == null) { + return true; + } + if((this.signature == null && dest.getSignature() != null) || + (this.signature != null && dest.getSignature() == null) ) { + return false; + } + + if(this.signature.size() != dest.getSignature().size()) { + return false; + } + + Iterator origIt = this.signature.iterator(); + Iterator destIt = dest.getSignature().iterator(); + while(origIt.hasNext()) { + ColumnInfo origColumn = origIt.next(); + ColumnInfo destColumn = destIt.next(); + + if(origColumn == null && destColumn == null) { + continue; + } + + if((origColumn == null && destColumn != null) || + (origColumn != null && destColumn == null) ) { + return false; + } + + if(!origColumn.equals(destColumn)) { + return false; + } + } + + return true; + } + + @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append('('); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java index a993de9..1aebe28 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java @@ -27,8 +27,10 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.Iterator; import java.util.Map; +import java.util.Set; import java.util.Timer; import java.util.TimerTask; import java.util.concurrent.TimeUnit; @@ -83,6 +85,8 @@ transient Deserializer scriptOutputDeserializer; transient volatile Throwable scriptError = null; transient RecordWriter scriptOutWriter = null; + // List of conf entries not to turn into env vars + transient Set blackListedConfEntries = null; static final String IO_EXCEPTION_BROKEN_PIPE_STRING = "Broken pipe"; static final String IO_EXCEPTION_STREAM_CLOSED = "Stream closed"; @@ -120,7 +124,8 @@ String safeEnvVarName(String name) { /** * Most UNIX implementations impose some limit on the total size of environment variables and - * size of strings. To fit in this limit we need sometimes to truncate strings. + * size of strings. To fit in this limit we need sometimes to truncate strings. Also, + * some values tend be long and are meaningless to scripts, so strain them out. * @param value environment variable value to check * @param name name of variable (used only for logging purposes) * @param truncate truncate value or not @@ -139,6 +144,23 @@ String safeEnvVarValue(String value, String name, boolean truncate) { return value; } + boolean blackListed(String name) { + if (blackListedConfEntries == null) { + blackListedConfEntries = new HashSet(); + if (hconf != null) { + String bl = hconf.get(HiveConf.ConfVars.HIVESCRIPT_ENV_BLACKLIST.toString()); + if (bl != null && bl.length() > 0) { + String[] bls = bl.split(","); + for (String b : bls) { + b.replaceAll(".", "_"); + blackListedConfEntries.add(b); + } + } + } + } + return blackListedConfEntries.contains(name); + } + /** * addJobConfToEnvironment is mostly shamelessly copied from hadoop streaming. Added additional * check on environment variable length @@ -148,13 +170,16 @@ void addJobConfToEnvironment(Configuration conf, Map env) { while (it.hasNext()) { Map.Entry en = it.next(); String name = en.getKey(); - // String value = (String)en.getValue(); // does not apply variable - // expansion - String value = conf.get(name); // does variable expansion - name = safeEnvVarName(name); - boolean truncate = conf.getBoolean(HiveConf.ConfVars.HIVESCRIPTTRUNCATEENV.toString(), false); - value = safeEnvVarValue(value, name, truncate); - env.put(name, value); + if (!blackListed(name)) { + // String value = (String)en.getValue(); // does not apply variable + // expansion + String value = conf.get(name); // does variable expansion + name = safeEnvVarName(name); + boolean truncate = conf + .getBoolean(HiveConf.ConfVars.HIVESCRIPTTRUNCATEENV.toString(), false); + value = safeEnvVarValue(value, name, truncate); + env.put(name, value); + } } } @@ -238,8 +263,8 @@ public File getAbsolutePath(String filename) { protected void initializeOp(Configuration hconf) throws HiveException { firstRow = true; - statsMap.put(Counter.DESERIALIZE_ERRORS, deserialize_error_count); - statsMap.put(Counter.SERIALIZE_ERRORS, serialize_error_count); + statsMap.put(Counter.DESERIALIZE_ERRORS.toString(), deserialize_error_count); + statsMap.put(Counter.SERIALIZE_ERRORS.toString(), serialize_error_count); try { this.hconf = hconf; @@ -285,6 +310,16 @@ void displayBrokenPipeInfo() { return; } + private transient String tableName; + private transient String partitionName ; + + @Override + public void setInputContext(String inputPath, String tableName, String partitionName) { + this.tableName = tableName; + this.partitionName = partitionName; + super.setInputContext(inputPath, tableName, partitionName); + } + @Override public void processOp(Object row, int tag) throws HiveException { // initialize the user's process only when you receive the first row @@ -313,10 +348,8 @@ public void processOp(Object row, int tag) throws HiveException { String[] wrappedCmdArgs = addWrapper(cmdArgs); LOG.info("Executing " + Arrays.asList(wrappedCmdArgs)); - LOG.info("tablename=" - + hconf.get(HiveConf.ConfVars.HIVETABLENAME.varname)); - LOG.info("partname=" - + hconf.get(HiveConf.ConfVars.HIVEPARTITIONNAME.varname)); + LOG.info("tablename=" + tableName); + LOG.info("partname=" + partitionName); LOG.info("alias=" + alias); ProcessBuilder pb = new ProcessBuilder(wrappedCmdArgs); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java index 42b546b..95d2d76 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java @@ -125,4 +125,31 @@ public boolean supportUnionRemoveOptimization() { public boolean acceptLimitPushdown() { return true; } + + /** + * Checks whether this select operator does something to the + * input tuples. + * + * @return if it is an identity select operator or not + */ + public boolean isIdentitySelect() { + //Safety check + if(this.getNumParent() != 1) { + return false; + } + + //Select * + if(this.getConf().isSelStarNoCompute() || + this.getConf().isSelectStar()) { + return true; + } + + //Check whether the have the same schema + if(!OperatorUtils.sameRowSchema(this, this.getParentOperators().get(0))) { + return false; + } + + return true; + } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java index 07efb8f..f089964 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java @@ -153,7 +153,9 @@ public void run() { partn.getInputFormatClass(), jc); InputSplit dummySplit = new FileSplit(file.getPath(), 0, 0, new String[] { partn.getLocation() }); - Object recordReader = inputFormat.getRecordReader(dummySplit, jc, Reporter.NULL); + org.apache.hadoop.mapred.RecordReader recordReader = + (org.apache.hadoop.mapred.RecordReader) + inputFormat.getRecordReader(dummySplit, jc, Reporter.NULL); StatsProvidingRecordReader statsRR; if (recordReader instanceof StatsProvidingRecordReader) { statsRR = (StatsProvidingRecordReader) recordReader; @@ -163,6 +165,7 @@ public void run() { numFiles += 1; statsAvailable = true; } + recordReader.close(); } } @@ -254,6 +257,7 @@ private int aggregateStats(ExecutorService threadPool) { numFiles += 1; statsAvailable = true; } + recordReader.close(); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java index 59c07c3..0f761e8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java @@ -80,7 +80,7 @@ protected void initializeOp(Configuration hconf) throws HiveException { for (int p = 0; p < parents; p++) { assert (parentFields[p].size() == columns); for (int c = 0; c < columns; c++) { - if (!columnTypeResolvers[c].update(parentFields[p].get(c) + if (!columnTypeResolvers[c].updateForUnionAll(parentFields[p].get(c) .getFieldObjectInspector())) { // checked in SemanticAnalyzer. Should not happen throw new HiveException("Incompatible types for union operator"); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index 99207f4..3f6c9eb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -416,7 +416,7 @@ private static BaseWork getBaseWork(Configuration conf, String name) { } gWorkMap.put(path, gWork); } else { - LOG.debug("Found plan in cache."); + LOG.debug("Found plan in cache for name: " + name); gWork = gWorkMap.get(path); } return gWork; @@ -437,20 +437,20 @@ private static BaseWork getBaseWork(Configuration conf, String name) { } } - public static Map> getScratchColumnVectorTypes(Configuration hiveConf) { + public static Map> getAllScratchColumnVectorTypeMaps(Configuration hiveConf) { BaseWork baseWork = getMapWork(hiveConf); if (baseWork == null) { baseWork = getReduceWork(hiveConf); } - return baseWork.getScratchColumnVectorTypes(); + return baseWork.getAllScratchColumnVectorTypeMaps(); } - public static Map> getScratchColumnMap(Configuration hiveConf) { + public static Map> getAllColumnVectorMaps(Configuration hiveConf) { BaseWork baseWork = getMapWork(hiveConf); if (baseWork == null) { baseWork = getReduceWork(hiveConf); } - return baseWork.getScratchColumnMap(); + return baseWork.getAllColumnVectorMaps(); } public static void setWorkflowAdjacencies(Configuration conf, QueryPlan plan) { @@ -1635,12 +1635,13 @@ public static void renameOrMoveFiles(FileSystem fs, Path src, Path dst) throws I * Group 6: copy [copy keyword] * Group 8: 2 [copy file index] */ + private static final String COPY_KEYWORD = "_copy_"; // copy keyword private static final Pattern COPY_FILE_NAME_TO_TASK_ID_REGEX = Pattern.compile("^.*?"+ // any prefix "([0-9]+)"+ // taskId "(_)"+ // separator "([0-9]{1,6})?"+ // attemptId (limited to 6 digits) - "((_)(\\Bcopy\\B)(_)"+ // copy keyword + "((_)(\\Bcopy\\B)(_)" + "([0-9]{1,6})$)?"+ // copy file index "(\\..*)?$"); // any suffix/file extension @@ -2035,6 +2036,15 @@ public static boolean isCopyFile(String filename) { return false; } + public static String getBucketFileNameFromPathSubString(String bucketName) { + try { + return bucketName.split(COPY_KEYWORD)[0]; + } catch (Exception e) { + e.printStackTrace(); + return bucketName; + } + } + public static String getNameMessage(Exception e) { return e.getClass().getName() + "(" + e.getMessage() + ")"; } @@ -2067,15 +2077,21 @@ public static String getResourceFiles(Configuration conf, SessionState.ResourceT public static ClassLoader getSessionSpecifiedClassLoader() { SessionState state = SessionState.get(); if (state == null || state.getConf() == null) { - LOG.debug("Hive Conf not found or Session not initiated, use thread based class loader instead"); + if (LOG.isDebugEnabled()) { + LOG.debug("Hive Conf not found or Session not initiated, use thread based class loader instead"); + } return JavaUtils.getClassLoader(); } ClassLoader sessionCL = state.getConf().getClassLoader(); - if (sessionCL != null){ - LOG.debug("Use session specified class loader"); + if (sessionCL != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Use session specified class loader"); + } return sessionCL; } - LOG.debug("Session specified class loader not found, use thread based class loader"); + if (LOG.isDebugEnabled()) { + LOG.debug("Session specified class loader not found, use thread based class loader"); + } return JavaUtils.getClassLoader(); } @@ -2363,6 +2379,32 @@ public static void copyTableJobPropertiesToConf(TableDesc tbl, JobConf job) { } } + /** + * Copies the storage handler proeprites configured for a table descriptor to a runtime job + * configuration. This differs from {@link #copyTablePropertiesToConf(org.apache.hadoop.hive.ql.plan.TableDesc, org.apache.hadoop.mapred.JobConf)} + * in that it does not allow parameters already set in the job to override the values from the + * table. This is important for setting the config up for reading, + * as the job may already have values in it from another table. + * @param tbl + * @param job + */ + public static void copyTablePropertiesToConf(TableDesc tbl, JobConf job) { + Properties tblProperties = tbl.getProperties(); + for(String name: tblProperties.stringPropertyNames()) { + String val = (String) tblProperties.get(name); + if (val != null) { + job.set(name, StringEscapeUtils.escapeJava(val)); + } + } + Map jobProperties = tbl.getJobProperties(); + if (jobProperties == null) { + return; + } + for (Map.Entry entry : jobProperties.entrySet()) { + job.set(entry.getKey(), entry.getValue()); + } + } + private static final Object INPUT_SUMMARY_LOCK = new Object(); /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java index 98cf2a7..4e3df75 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java @@ -56,6 +56,8 @@ import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.exec.tez.TezSessionState; +import org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolManager; import org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat; import org.apache.hadoop.hive.ql.io.HiveKey; import org.apache.hadoop.hive.ql.io.HiveOutputFormatImpl; @@ -416,6 +418,13 @@ public int execute(DriverContext driverContext) { Utilities.createTmpDirs(job, mWork); Utilities.createTmpDirs(job, rWork); + SessionState ss = SessionState.get(); + if (HiveConf.getVar(job, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez") + && ss != null) { + TezSessionState session = ss.getTezSession(); + TezSessionPoolManager.getInstance().close(session, true); + } + // Finally SUBMIT the JOB! rj = jc.submitJob(job); // replace it back diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java index f188e69..b6a7388 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java @@ -26,8 +26,11 @@ import java.util.List; import java.util.Map; +import org.apache.hadoop.conf.Configuration; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.ql.exec.FetchOperator; import org.apache.hadoop.hive.ql.exec.MapOperator; import org.apache.hadoop.hive.ql.exec.MapredContext; @@ -72,9 +75,6 @@ private static boolean done; // used to log memory usage periodically - public static MemoryMXBean memoryMXBean; - private long numRows = 0; - private long nextCntr = 1; private MapredLocalWork localWork = null; private boolean isLogInfoEnabled = false; @@ -84,8 +84,6 @@ public void configure(JobConf job) { execContext = new ExecMapperContext(job); // Allocate the bean at the beginning - - memoryMXBean = ManagementFactory.getMemoryMXBean(); - l4j.info("maximum memory = " + memoryMXBean.getHeapMemoryUsage().getMax()); isLogInfoEnabled = l4j.isInfoEnabled(); @@ -176,15 +174,6 @@ public void map(Object key, Object value, OutputCollector output, // Since there is no concept of a group, we don't invoke // startGroup/endGroup for a mapper mo.process((Writable)value); - if (isLogInfoEnabled) { - numRows++; - if (numRows == nextCntr) { - long used_memory = memoryMXBean.getHeapMemoryUsage().getUsed(); - l4j.info("ExecMapper: processing " + numRows - + " rows: used memory = " + used_memory); - nextCntr = getNextCntr(numRows); - } - } } } catch (Throwable e) { abort = true; @@ -198,18 +187,6 @@ public void map(Object key, Object value, OutputCollector output, } } - - private long getNextCntr(long cntr) { - // A very simple counter to keep track of number of rows processed by the - // reducer. It dumps - // every 1 million times, and quickly before that - if (cntr >= 1000000) { - return cntr + 1000000; - } - - return 10 * cntr; - } - @Override public void close() { // No row was processed @@ -245,13 +222,7 @@ public void close() { } } - if (isLogInfoEnabled) { - long used_memory = memoryMXBean.getHeapMemoryUsage().getUsed(); - l4j.info("ExecMapper: processed " + numRows + " rows: used memory = " - + used_memory); - } - - ReportStats rps = new ReportStats(rp); + ReportStats rps = new ReportStats(rp, jc); mo.preorderMap(rps); return; } catch (Exception e) { @@ -288,17 +259,21 @@ public static void setDone(boolean done) { */ public static class ReportStats implements Operator.OperatorFunc { private final Reporter rp; + private final Configuration conf; + private final String groupName; - public ReportStats(Reporter rp) { + public ReportStats(Reporter rp, Configuration conf) { this.rp = rp; + this.conf = conf; + this.groupName = HiveConf.getVar(conf, HiveConf.ConfVars.HIVECOUNTERGROUP); } @Override public void func(Operator op) { - Map, Long> opStats = op.getStats(); - for (Map.Entry, Long> e : opStats.entrySet()) { + Map opStats = op.getStats(); + for (Map.Entry e : opStats.entrySet()) { if (rp != null) { - rp.incrCounter(e.getKey(), e.getValue()); + rp.incrCounter(groupName, e.getKey(), e.getValue()); } } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecReducer.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecReducer.java index 9cd8b56..8c2ba6c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecReducer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecReducer.java @@ -70,8 +70,6 @@ private static final boolean isTraceEnabled = LOG.isTraceEnabled(); private static final String PLAN_KEY = "__REDUCE_PLAN__"; - // used to log memory usage periodically - private final MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean(); // Input value serde needs to be an array to support different SerDe // for different tags private final Deserializer[] inputValueDeserializer = new Deserializer[Byte.MAX_VALUE]; @@ -86,8 +84,6 @@ private Reporter rp; private boolean abort = false; private boolean isTagged = false; - private long cntr = 0; - private long nextCntr = 1; private TableDesc keyTableDesc; private TableDesc[] valueTableDesc; private ObjectInspector[] rowObjectInspector; @@ -103,8 +99,6 @@ public void configure(JobConf job) { ObjectInspector keyObjectInspector; if (isInfoEnabled) { - LOG.info("maximum memory = " + memoryMXBean.getHeapMemoryUsage().getMax()); - try { LOG.info("conf classpath = " + Arrays.asList(((URLClassLoader) job.getClassLoader()).getURLs())); @@ -245,17 +239,7 @@ public void reduce(Object key, Iterator values, OutputCollector output, row.clear(); row.add(keyObject); row.add(valueObject[tag]); - if (isInfoEnabled) { - cntr++; - if (cntr == nextCntr) { - long used_memory = memoryMXBean.getHeapMemoryUsage().getUsed(); - if (isInfoEnabled) { - LOG.info("ExecReducer: processing " + cntr - + " rows: used memory = " + used_memory); - } - nextCntr = getNextCntr(cntr); - } - } + try { reducer.processOp(row, tag); } catch (Exception e) { @@ -283,17 +267,6 @@ public void reduce(Object key, Iterator values, OutputCollector output, } } - private long getNextCntr(long cntr) { - // A very simple counter to keep track of number of rows processed by the - // reducer. It dumps - // every 1 million times, and quickly before that - if (cntr >= 1000000) { - return cntr + 1000000; - } - - return 10 * cntr; - } - @Override public void close() { @@ -310,13 +283,9 @@ public void close() { } reducer.endGroup(); } - if (isInfoEnabled) { - LOG.info("ExecReducer: processed " + cntr + " rows: used memory = " - + memoryMXBean.getHeapMemoryUsage().getUsed()); - } reducer.close(abort); - ReportStats rps = new ReportStats(rp); + ReportStats rps = new ReportStats(rp, jc); reducer.preorderMap(rps); } catch (Exception e) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java index 79da5a0..0676312 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java @@ -238,6 +238,18 @@ public int executeInChildVM(DriverContext driverContext) { variables.put(HADOOP_OPTS_KEY, hadoopOpts); } + //For Windows OS, we need to pass HIVE_HADOOP_CLASSPATH Java parameter while starting + //Hiveserver2 using "-hiveconf hive.hadoop.classpath=%HIVE_LIB%". This is to combine path(s). + if (HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_HADOOP_CLASSPATH)!= null) + { + if (variables.containsKey("HADOOP_CLASSPATH")) + { + variables.put("HADOOP_CLASSPATH", variables.get("HADOOP_CLASSPATH") + ";" + HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_HADOOP_CLASSPATH)); + } else { + variables.put("HADOOP_CLASSPATH", HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_HADOOP_CLASSPATH)); + } + } + if(variables.containsKey(MapRedTask.HIVE_DEBUG_RECURSIVE)) { MapRedTask.configureDebugVariablesForChildJVM(variables); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java index 0bb554a..792a934 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java @@ -196,7 +196,7 @@ public void close() { logCloseInfo(); } - ReportStats rps = new ReportStats(rp); + ReportStats rps = new ReportStats(rp, jc); mo.preorderMap(rps); return; } catch (Exception e) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java index 3e6ca56..21ac7ab 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java @@ -410,7 +410,7 @@ public void close() { } reducer.close(abort); - ReportStats rps = new ReportStats(rp); + ReportStats rps = new ReportStats(rp, jc); reducer.preorderMap(rps); } catch (Exception e) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/counter/SparkCounters.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/counter/SparkCounters.java index 447d964..174621a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/counter/SparkCounters.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/counter/SparkCounters.java @@ -23,9 +23,11 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.exec.FileSinkOperator; import org.apache.hadoop.hive.ql.exec.FilterOperator; import org.apache.hadoop.hive.ql.exec.JoinOperator; import org.apache.hadoop.hive.ql.exec.MapOperator; +import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.ScriptOperator; import org.apache.hadoop.mapreduce.util.ResourceBundles; @@ -61,11 +63,18 @@ public SparkCounters(JavaSparkContext javaSparkContext, Configuration hiveConf) * pre-define all needed Counters here. */ private void initializeSparkCounters() { - createCounter(HiveConf.getVar(hiveConf, HiveConf.ConfVars.HIVECOUNTERGROUP), - Operator.HIVECOUNTERCREATEDFILES); + String groupName = HiveConf.getVar(hiveConf, HiveConf.ConfVars.HIVECOUNTERGROUP); + createCounter(groupName, Operator.HIVECOUNTERCREATEDFILES); + createCounter(groupName, MapOperator.Counter.DESERIALIZE_ERRORS); + createCounter(groupName, MapOperator.Counter.RECORDS_IN); + createCounter(groupName, FileSinkOperator.Counter.RECORDS_OUT); + createCounter(groupName, ReduceSinkOperator.Counter.RECORDS_OUT_INTERMEDIATE); + createCounter(groupName, ScriptOperator.Counter.DESERIALIZE_ERRORS); + createCounter(groupName, ScriptOperator.Counter.SERIALIZE_ERRORS); + createCounter(groupName, JoinOperator.SkewkeyTableCounter.SKEWJOINFOLLOWUPJOBS); + // TODO remove? changed due to HIVE-8429 createCounter(MapOperator.Counter.DESERIALIZE_ERRORS); - createCounter(FilterOperator.Counter.FILTERED); - createCounter(FilterOperator.Counter.PASSED); + createCounter(MapOperator.Counter.RECORDS_IN); createCounter(ScriptOperator.Counter.DESERIALIZE_ERRORS); createCounter(ScriptOperator.Counter.SERIALIZE_ERRORS); createCounter(JoinOperator.SkewkeyTableCounter.SKEWJOINFOLLOWUPJOBS); @@ -75,6 +84,10 @@ public void createCounter(Enum key) { createCounter(key.getDeclaringClass().getName(), key.name()); } + public void createCounter(String groupName, Enum key) { + createCounter(groupName, key.name(), 0L); + } + public void createCounter(String groupName, String counterName) { createCounter(groupName, counterName, 0L); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java index 848be26..297ce44 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java @@ -21,16 +21,23 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Set; import java.util.TreeMap; +import java.util.TreeSet; +import com.google.common.collect.LinkedListMultimap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.plan.TezWork.VertexType; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.io.DataOutputBuffer; @@ -38,6 +45,7 @@ import org.apache.hadoop.mapred.FileSplit; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.split.TezGroupedSplit; import org.apache.hadoop.mapreduce.split.TezMapReduceSplitsGrouper; import org.apache.tez.common.TezUtils; import org.apache.tez.dag.api.EdgeProperty; @@ -67,17 +75,41 @@ import com.google.protobuf.ByteString; /* - * Only works with old mapred API - * Will only work with a single MRInput for now. + * This is the central piece for Bucket Map Join and SMB join. It has the following + * responsibilities: + * 1. Group incoming splits based on bucketing. + * 2. Generate new serialized events for the grouped splits. + * 3. Create a routing table for the bucket map join and send a serialized version as payload + * for the EdgeManager. + * 4. For SMB join, generate a grouping according to bucketing for the "small" table side. */ public class CustomPartitionVertex extends VertexManagerPlugin { + public class PathComparatorForSplit implements Comparator { + + @Override + public int compare(InputSplit inp1, InputSplit inp2) { + FileSplit fs1 = (FileSplit) inp1; + FileSplit fs2 = (FileSplit) inp2; + + int retval = fs1.getPath().compareTo(fs2.getPath()); + if (retval != 0) { + return retval; + } + + if (fs1.getStart() != fs2.getStart()) { + return (int) (fs1.getStart() - fs2.getStart()); + } + + return 0; + } + } + private static final Log LOG = LogFactory.getLog(CustomPartitionVertex.class.getName()); VertexManagerPluginContext context; private InputConfigureVertexTasksEvent configureVertexTaskEvent; - private List dataInformationEvents; private int numBuckets = -1; private Configuration conf = null; private final SplitGrouper grouper = new SplitGrouper(); @@ -89,6 +121,13 @@ private final Map> inputToGroupedSplitMap = new HashMap>(); + private int numInputsAffectingRootInputSpecUpdate = 1; + private int numInputsSeenSoFar = 0; + private final Map emMap = Maps.newHashMap(); + private final List finalSplits = Lists.newLinkedList(); + private final Map inputNameInputSpecMap = + new HashMap(); + public CustomPartitionVertex(VertexManagerPluginContext context) { super(context); } @@ -108,12 +147,13 @@ public void initialize() { this.numBuckets = vertexConf.getNumBuckets(); this.mainWorkName = vertexConf.getInputName(); this.vertexType = vertexConf.getVertexType(); + this.numInputsAffectingRootInputSpecUpdate = vertexConf.getNumInputs(); } @Override public void onVertexStarted(Map> completions) { int numTasks = context.getVertexNumTasks(context.getVertexName()); - List scheduledTasks = + List scheduledTasks = new ArrayList(numTasks); for (int i = 0; i < numTasks; ++i) { scheduledTasks.add(new VertexManagerPluginContext.TaskWithLocationHint(new Integer(i), null)); @@ -133,8 +173,8 @@ public void onVertexManagerEventReceived(VertexManagerEvent vmEvent) { @Override public void onRootVertexInitialized(String inputName, InputDescriptor inputDescriptor, List events) { + numInputsSeenSoFar++; LOG.info("On root vertex initialized " + inputName); - try { // This is using the payload from the RootVertexInitializer corresponding // to InputName. Ideally it should be using it's own configuration class - @@ -168,20 +208,21 @@ public void onRootVertexInitialized(String inputName, InputDescriptor inputDescr } boolean dataInformationEventSeen = false; - Map> pathFileSplitsMap = new TreeMap>(); + Map> pathFileSplitsMap = new TreeMap>(); for (Event event : events) { if (event instanceof InputConfigureVertexTasksEvent) { // No tasks should have been started yet. Checked by initial state // check. + LOG.info("Got a input configure vertex event for input: " + inputName); Preconditions.checkState(dataInformationEventSeen == false); InputConfigureVertexTasksEvent cEvent = (InputConfigureVertexTasksEvent) event; // The vertex cannot be configured until all DataEvents are seen - to // build the routing table. configureVertexTaskEvent = cEvent; - dataInformationEvents = - Lists.newArrayListWithCapacity(configureVertexTaskEvent.getNumTasks()); + LOG.info("Configure task for input name: " + inputName + " num tasks: " + + configureVertexTaskEvent.getNumTasks()); } if (event instanceof InputUpdatePayloadEvent) { // this event can never occur. If it does, fail. @@ -189,22 +230,26 @@ public void onRootVertexInitialized(String inputName, InputDescriptor inputDescr } else if (event instanceof InputDataInformationEvent) { dataInformationEventSeen = true; InputDataInformationEvent diEvent = (InputDataInformationEvent) event; - dataInformationEvents.add(diEvent); FileSplit fileSplit; try { fileSplit = getFileSplitFromEvent(diEvent); } catch (IOException e) { throw new RuntimeException("Failed to get file split for event: " + diEvent); } - List fsList = pathFileSplitsMap.get(fileSplit.getPath().getName()); + Set fsList = + pathFileSplitsMap.get(Utilities.getBucketFileNameFromPathSubString(fileSplit.getPath() + .getName())); if (fsList == null) { - fsList = new ArrayList(); - pathFileSplitsMap.put(fileSplit.getPath().getName(), fsList); + fsList = new TreeSet(new PathComparatorForSplit()); + pathFileSplitsMap.put( + Utilities.getBucketFileNameFromPathSubString(fileSplit.getPath().getName()), fsList); } fsList.add(fileSplit); } } + LOG.info("Path file splits map for input name: " + inputName + " is " + pathFileSplitsMap); + Multimap bucketToInitialSplitMap = getBucketSplitMapForPath(pathFileSplitsMap); @@ -217,77 +262,144 @@ public void onRootVertexInitialized(String inputName, InputDescriptor inputDescr int availableSlots = totalResource / taskResource; - LOG.info("Grouping splits. " + availableSlots + " available slots, " + waves + " waves."); + LOG.info("Grouping splits. " + availableSlots + " available slots, " + waves + + " waves. Bucket initial splits map: " + bucketToInitialSplitMap); JobConf jobConf = new JobConf(conf); ShimLoader.getHadoopShims().getMergedCredentials(jobConf); Multimap bucketToGroupedSplitMap = HashMultimap. create(); - for (Integer key : bucketToInitialSplitMap.keySet()) { - InputSplit[] inputSplitArray = - (bucketToInitialSplitMap.get(key).toArray(new InputSplit[0])); - Multimap groupedSplit = - HiveSplitGenerator.generateGroupedSplits(jobConf, conf, inputSplitArray, waves, - availableSlots, inputName); - bucketToGroupedSplitMap.putAll(key, groupedSplit.values()); - } - - LOG.info("We have grouped the splits into " + bucketToGroupedSplitMap); - if ((mainWorkName.isEmpty() == false) && (mainWorkName.compareTo(inputName) != 0)) { + boolean secondLevelGroupingDone = false; + if ((mainWorkName.isEmpty()) || (inputName.compareTo(mainWorkName) == 0)) { + for (Integer key : bucketToInitialSplitMap.keySet()) { + InputSplit[] inputSplitArray = + (bucketToInitialSplitMap.get(key).toArray(new InputSplit[0])); + HiveSplitGenerator hiveSplitGenerator = new HiveSplitGenerator(); + Multimap groupedSplit = + hiveSplitGenerator.generateGroupedSplits(jobConf, conf, inputSplitArray, waves, + availableSlots, inputName, mainWorkName.isEmpty()); + if (mainWorkName.isEmpty() == false) { + Multimap singleBucketToGroupedSplit = + HashMultimap. create(); + singleBucketToGroupedSplit.putAll(key, groupedSplit.values()); + groupedSplit = + grouper.group(jobConf, singleBucketToGroupedSplit, availableSlots, + HiveConf.getFloatVar(conf, HiveConf.ConfVars.TEZ_SMB_NUMBER_WAVES)); + secondLevelGroupingDone = true; + } + bucketToGroupedSplitMap.putAll(key, groupedSplit.values()); + } + processAllEvents(inputName, bucketToGroupedSplitMap, secondLevelGroupingDone); + } else { + // do not group across files in case of side work because there is only 1 KV reader per + // grouped split. This would affect SMB joins where we want to find the smallest key in + // all the bucket files. + for (Integer key : bucketToInitialSplitMap.keySet()) { + HiveSplitGenerator hiveSplitGenerator = new HiveSplitGenerator(); + InputSplit[] inputSplitArray = + (bucketToInitialSplitMap.get(key).toArray(new InputSplit[0])); + Multimap groupedSplit = + hiveSplitGenerator.generateGroupedSplits(jobConf, conf, inputSplitArray, waves, + availableSlots, inputName, false); + bucketToGroupedSplitMap.putAll(key, groupedSplit.values()); + } /* - * this is the small table side. In case of SMB join, we may need to send each split to the + * this is the small table side. In case of SMB join, we need to send each split to the * corresponding bucket-based task on the other side. In case a split needs to go to * multiple downstream tasks, we need to clone the event and send it to the right * destination. */ - processAllSideEvents(inputName, bucketToGroupedSplitMap); - } else { - processAllEvents(inputName, bucketToGroupedSplitMap); + LOG.info("This is the side work - multi-mr work."); + processAllSideEventsSetParallelism(inputName, bucketToGroupedSplitMap); } } catch (Exception e) { throw new RuntimeException(e); } } - private void processAllSideEvents(String inputName, + private void processAllSideEventsSetParallelism(String inputName, Multimap bucketToGroupedSplitMap) throws IOException { // the bucket to task map should have been setup by the big table. + LOG.info("Processing events for input " + inputName); if (bucketToTaskMap.isEmpty()) { + LOG.info("We don't have a routing table yet. Will need to wait for the main input" + + " initialization"); inputToGroupedSplitMap.put(inputName, bucketToGroupedSplitMap); return; } + processAllSideEvents(inputName, bucketToGroupedSplitMap); + setVertexParallelismAndRootInputSpec(inputNameInputSpecMap); + } + + private void processAllSideEvents(String inputName, + Multimap bucketToGroupedSplitMap) throws IOException { List taskEvents = new ArrayList(); + LOG.info("We have a routing table and we are going to set the destination tasks for the" + + " multi mr inputs. " + bucketToTaskMap); + + Integer[] numSplitsForTask = new Integer[taskCount]; + + Multimap bucketToSerializedSplitMap = LinkedListMultimap.create(); + + // Create the list of serialized splits for each bucket. for (Entry> entry : bucketToGroupedSplitMap.asMap().entrySet()) { + for (InputSplit split : entry.getValue()) { + MRSplitProto serializedSplit = MRInputHelpers.createSplitProto(split); + ByteBuffer bs = serializedSplit.toByteString().asReadOnlyByteBuffer(); + bucketToSerializedSplitMap.put(entry.getKey(), bs); + } + } + + for (Entry> entry : bucketToSerializedSplitMap.asMap().entrySet()) { Collection destTasks = bucketToTaskMap.get(entry.getKey()); for (Integer task : destTasks) { - for (InputSplit split : entry.getValue()) { - MRSplitProto serializedSplit = MRInputHelpers.createSplitProto(split); + int count = 0; + for (ByteBuffer buf : entry.getValue()) { + count++; InputDataInformationEvent diEvent = - InputDataInformationEvent.createWithSerializedPayload(task, serializedSplit - .toByteString().asReadOnlyByteBuffer()); + InputDataInformationEvent.createWithSerializedPayload(count, buf); diEvent.setTargetIndex(task); taskEvents.add(diEvent); } + numSplitsForTask[task] = count; } } + inputNameInputSpecMap.put(inputName, + InputSpecUpdate.createPerTaskInputSpecUpdate(Arrays.asList(numSplitsForTask))); + + LOG.info("For input name: " + inputName + " task events size is " + taskEvents.size()); + context.addRootInputEvents(inputName, taskEvents); } private void processAllEvents(String inputName, - Multimap bucketToGroupedSplitMap) throws IOException { + Multimap bucketToGroupedSplitMap, boolean secondLevelGroupingDone) + throws IOException { - List finalSplits = Lists.newLinkedList(); + int totalInputsCount = 0; + List numSplitsForTask = new ArrayList(); for (Entry> entry : bucketToGroupedSplitMap.asMap().entrySet()) { int bucketNum = entry.getKey(); Collection initialSplits = entry.getValue(); finalSplits.addAll(initialSplits); - for (int i = 0; i < initialSplits.size(); i++) { + for (InputSplit inputSplit : initialSplits) { bucketToTaskMap.put(bucketNum, taskCount); + if (secondLevelGroupingDone) { + TezGroupedSplit groupedSplit = (TezGroupedSplit) inputSplit; + numSplitsForTask.add(groupedSplit.getGroupedSplits().size()); + totalInputsCount += groupedSplit.getGroupedSplits().size(); + } else { + numSplitsForTask.add(1); + totalInputsCount += 1; + } taskCount++; } } + inputNameInputSpecMap.put(inputName, + InputSpecUpdate.createPerTaskInputSpecUpdate(numSplitsForTask)); + // Construct the EdgeManager descriptor to be used by all edges which need // the routing table. EdgeManagerPluginDescriptor hiveEdgeManagerDesc = null; @@ -297,7 +409,6 @@ private void processAllEvents(String inputName, UserPayload payload = getBytePayload(bucketToTaskMap); hiveEdgeManagerDesc.setUserPayload(payload); } - Map emMap = Maps.newHashMap(); // Replace the edge manager for all vertices which have routing type custom. for (Entry edgeEntry : context.getInputVertexEdgeProperties().entrySet()) { @@ -308,42 +419,67 @@ private void processAllEvents(String inputName, } } - LOG.info("Task count is " + taskCount); + LOG.info("Task count is " + taskCount + " for input name: " + inputName); - List taskEvents = - Lists.newArrayListWithCapacity(finalSplits.size()); + List taskEvents = Lists.newArrayListWithCapacity(totalInputsCount); // Re-serialize the splits after grouping. int count = 0; for (InputSplit inputSplit : finalSplits) { - MRSplitProto serializedSplit = MRInputHelpers.createSplitProto(inputSplit); - InputDataInformationEvent diEvent = InputDataInformationEvent.createWithSerializedPayload( - count, serializedSplit.toByteString().asReadOnlyByteBuffer()); - diEvent.setTargetIndex(count); + if (secondLevelGroupingDone) { + TezGroupedSplit tezGroupedSplit = (TezGroupedSplit)inputSplit; + for (InputSplit subSplit : tezGroupedSplit.getGroupedSplits()) { + if ((subSplit instanceof TezGroupedSplit) == false) { + throw new IOException("Unexpected split type found: " + + subSplit.getClass().getCanonicalName()); + } + MRSplitProto serializedSplit = MRInputHelpers.createSplitProto(subSplit); + InputDataInformationEvent diEvent = + InputDataInformationEvent.createWithSerializedPayload(count, serializedSplit + .toByteString().asReadOnlyByteBuffer()); + diEvent.setTargetIndex(count); + taskEvents.add(diEvent); + } + } else { + MRSplitProto serializedSplit = MRInputHelpers.createSplitProto(inputSplit); + InputDataInformationEvent diEvent = + InputDataInformationEvent.createWithSerializedPayload(count, serializedSplit + .toByteString().asReadOnlyByteBuffer()); + diEvent.setTargetIndex(count); + taskEvents.add(diEvent); + } count++; - taskEvents.add(diEvent); - } - - // Replace the Edge Managers - Map rootInputSpecUpdate = - new HashMap(); - rootInputSpecUpdate.put( - inputName, - InputSpecUpdate.getDefaultSinglePhysicalInputSpecUpdate()); - if ((mainWorkName.compareTo(inputName) == 0) || (mainWorkName.isEmpty())) { - context.setVertexParallelism( - taskCount, - VertexLocationHint.create(grouper.createTaskLocationHints(finalSplits - .toArray(new InputSplit[finalSplits.size()]))), emMap, rootInputSpecUpdate); } // Set the actual events for the tasks. + LOG.info("For input name: " + inputName + " task events size is " + taskEvents.size()); context.addRootInputEvents(inputName, taskEvents); if (inputToGroupedSplitMap.isEmpty() == false) { for (Entry> entry : inputToGroupedSplitMap.entrySet()) { processAllSideEvents(entry.getKey(), entry.getValue()); } + setVertexParallelismAndRootInputSpec(inputNameInputSpecMap); inputToGroupedSplitMap.clear(); } + + // Only done when it is a bucket map join only no SMB. + if (numInputsAffectingRootInputSpecUpdate == 1) { + setVertexParallelismAndRootInputSpec(inputNameInputSpecMap); + } + } + + private void + setVertexParallelismAndRootInputSpec(Map rootInputSpecUpdate) + throws IOException { + if (numInputsAffectingRootInputSpecUpdate != numInputsSeenSoFar) { + return; + } + + LOG.info("Setting vertex parallelism since we have seen all inputs."); + + context.setVertexParallelism(taskCount, VertexLocationHint.create(grouper + .createTaskLocationHints(finalSplits.toArray(new InputSplit[finalSplits.size()]))), emMap, + rootInputSpecUpdate); + finalSplits.clear(); } UserPayload getBytePayload(Multimap routingTable) throws IOException { @@ -377,14 +513,14 @@ private FileSplit getFileSplitFromEvent(InputDataInformationEvent event) throws * This method generates the map of bucket to file splits. */ private Multimap getBucketSplitMapForPath( - Map> pathFileSplitsMap) { + Map> pathFileSplitsMap) { int bucketNum = 0; Multimap bucketToInitialSplitMap = ArrayListMultimap. create(); - for (Map.Entry> entry : pathFileSplitsMap.entrySet()) { + for (Map.Entry> entry : pathFileSplitsMap.entrySet()) { int bucketId = bucketNum % numBuckets; for (FileSplit fsplit : entry.getValue()) { bucketToInitialSplitMap.put(bucketId, fsplit); @@ -392,6 +528,11 @@ private FileSplit getFileSplitFromEvent(InputDataInformationEvent event) throws bucketNum++; } + // this is just for SMB join use-case. The numBuckets would be equal to that of the big table + // and the small table could have lesser number of buckets. In this case, we want to send the + // data from the right buckets to the big table side. For e.g. Big table has 8 buckets and small + // table has 4 buckets, bucket 0 of small table needs to be sent to bucket 4 of the big table as + // well. if (bucketNum < numBuckets) { int loopedBucketId = 0; for (; bucketNum < numBuckets; bucketNum++) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomVertexConfiguration.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomVertexConfiguration.java index 4829f92..5dd7bf3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomVertexConfiguration.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomVertexConfiguration.java @@ -29,20 +29,31 @@ * This class is the payload for custom vertex. It serializes and de-serializes * @numBuckets: the number of buckets of the "big table" * @vertexType: this is the type of vertex and differentiates between bucket map join and SMB joins - * @inputName: This is the name of the input. Used in case of SMB joins + * @numInputs: The number of inputs that are directly connected to the vertex (MRInput/MultiMRInput). + * In case of bucket map join, it is always 1. + * @inputName: This is the name of the input. Used in case of SMB joins. Empty in case of BucketMapJoin */ public class CustomVertexConfiguration implements Writable { private int numBuckets; private VertexType vertexType = VertexType.AUTO_INITIALIZED_EDGES; + private int numInputs; private String inputName; public CustomVertexConfiguration() { } - public CustomVertexConfiguration(int numBuckets, VertexType vertexType, String inputName) { + // this is the constructor to use for the Bucket map join case. + public CustomVertexConfiguration(int numBuckets, VertexType vertexType) { + this(numBuckets, vertexType, "", 1); + } + + // this is the constructor to use for SMB. + public CustomVertexConfiguration(int numBuckets, VertexType vertexType, String inputName, + int numInputs) { this.numBuckets = numBuckets; this.vertexType = vertexType; + this.numInputs = numInputs; this.inputName = inputName; } @@ -50,6 +61,7 @@ public CustomVertexConfiguration(int numBuckets, VertexType vertexType, String i public void write(DataOutput out) throws IOException { out.writeInt(this.vertexType.ordinal()); out.writeInt(this.numBuckets); + out.writeInt(numInputs); out.writeUTF(inputName); } @@ -57,6 +69,7 @@ public void write(DataOutput out) throws IOException { public void readFields(DataInput in) throws IOException { this.vertexType = VertexType.values()[in.readInt()]; this.numBuckets = in.readInt(); + this.numInputs = in.readInt(); this.inputName = in.readUTF(); } @@ -71,4 +84,8 @@ public VertexType getVertexType() { public String getInputName() { return inputName; } + + public int getNumInputs() { + return numInputs; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java index 0a0418e..0d3c29d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java @@ -20,8 +20,6 @@ import com.google.common.base.Function; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; -import com.google.protobuf.ByteString; - import javax.security.auth.login.LoginException; import java.io.FileNotFoundException; import java.io.IOException; @@ -49,7 +47,7 @@ import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.ErrorMsg; -import org.apache.hadoop.hive.ql.exec.CommonMergeJoinOperator; +import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.exec.mr.ExecMapper; import org.apache.hadoop.hive.ql.exec.mr.ExecReducer; @@ -110,16 +108,13 @@ import org.apache.tez.dag.api.VertexGroup; import org.apache.tez.dag.api.VertexManagerPluginDescriptor; import org.apache.tez.dag.library.vertexmanager.ShuffleVertexManager; -import org.apache.tez.mapreduce.common.MRInputAMSplitGenerator; import org.apache.tez.mapreduce.hadoop.MRHelpers; import org.apache.tez.mapreduce.hadoop.MRInputHelpers; import org.apache.tez.mapreduce.hadoop.MRJobConfig; -import org.apache.tez.mapreduce.input.MRInput; import org.apache.tez.mapreduce.input.MRInputLegacy; import org.apache.tez.mapreduce.input.MultiMRInput; import org.apache.tez.mapreduce.output.MROutput; import org.apache.tez.mapreduce.partition.MRPartitioner; -import org.apache.tez.mapreduce.protos.MRRuntimeProtos; import org.apache.tez.runtime.library.api.TezRuntimeConfiguration; import org.apache.tez.runtime.library.common.comparator.TezBytesComparator; import org.apache.tez.runtime.library.common.serializer.TezBytesWritableSerialization; @@ -178,6 +173,8 @@ private void addCredentials(ReduceWork reduceWork, DAG dag) { private JobConf initializeVertexConf(JobConf baseConf, Context context, MapWork mapWork) { JobConf conf = new JobConf(baseConf); + conf.set(Operator.CONTEXT_NAME_KEY, mapWork.getName()); + if (mapWork.getNumMapTasks() != null) { // Is this required ? conf.setInt(MRJobConfig.NUM_MAPS, mapWork.getNumMapTasks().intValue()); @@ -268,8 +265,7 @@ public GroupInputEdge createEdge(VertexGroup group, JobConf vConf, Vertex w, case CUSTOM_EDGE: { mergeInputClass = ConcatenatedMergedKeyValueInput.class; int numBuckets = edgeProp.getNumBuckets(); - CustomVertexConfiguration vertexConf = - new CustomVertexConfiguration(numBuckets, vertexType, ""); + CustomVertexConfiguration vertexConf = new CustomVertexConfiguration(numBuckets, vertexType); DataOutputBuffer dob = new DataOutputBuffer(); vertexConf.write(dob); VertexManagerPluginDescriptor desc = @@ -314,8 +310,7 @@ public Edge createEdge(JobConf vConf, Vertex v, Vertex w, TezEdgeProperty edgePr switch(edgeProp.getEdgeType()) { case CUSTOM_EDGE: { int numBuckets = edgeProp.getNumBuckets(); - CustomVertexConfiguration vertexConf = - new CustomVertexConfiguration(numBuckets, vertexType, ""); + CustomVertexConfiguration vertexConf = new CustomVertexConfiguration(numBuckets, vertexType); DataOutputBuffer dob = new DataOutputBuffer(); vertexConf.write(dob); VertexManagerPluginDescriptor desc = VertexManagerPluginDescriptor.create( @@ -340,7 +335,6 @@ public Edge createEdge(JobConf vConf, Vertex v, Vertex w, TezEdgeProperty edgePr /* * Helper function to create an edge property from an edge type. */ - @SuppressWarnings("rawtypes") private EdgeProperty createEdgeProperty(TezEdgeProperty edgeProp, Configuration conf) throws IOException { MRHelpers.translateMRConfToTez(conf); @@ -431,8 +425,9 @@ public static Resource getContainerResource(Configuration conf) { int memory = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCONTAINERSIZE) > 0 ? HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCONTAINERSIZE) : conf.getInt(MRJobConfig.MAP_MEMORY_MB, MRJobConfig.DEFAULT_MAP_MEMORY_MB); - int cpus = conf.getInt(MRJobConfig.MAP_CPU_VCORES, - MRJobConfig.DEFAULT_MAP_CPU_VCORES); + int cpus = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCPUVCORES) > 0 ? + HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCPUVCORES) : + conf.getInt(MRJobConfig.MAP_CPU_VCORES, MRJobConfig.DEFAULT_MAP_CPU_VCORES); return Resource.newInstance(memory, cpus); } @@ -452,17 +447,29 @@ public static Resource getContainerResource(Configuration conf) { */ private String getContainerJavaOpts(Configuration conf) { String javaOpts = HiveConf.getVar(conf, HiveConf.ConfVars.HIVETEZJAVAOPTS); - if (javaOpts != null && !javaOpts.isEmpty()) { - String logLevel = HiveConf.getVar(conf, HiveConf.ConfVars.HIVETEZLOGLEVEL); - List logProps = Lists.newArrayList(); - TezUtils.addLog4jSystemProperties(logLevel, logProps); - StringBuilder sb = new StringBuilder(); - for (String str : logProps) { - sb.append(str).append(" "); + + String logLevel = HiveConf.getVar(conf, HiveConf.ConfVars.HIVETEZLOGLEVEL); + List logProps = Lists.newArrayList(); + TezUtils.addLog4jSystemProperties(logLevel, logProps); + StringBuilder sb = new StringBuilder(); + for (String str : logProps) { + sb.append(str).append(" "); + } + logLevel = sb.toString(); + + if (HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCONTAINERSIZE) > 0) { + if (javaOpts != null) { + return javaOpts + " " + logLevel; + } else { + return logLevel; + } + } else { + if (javaOpts != null && !javaOpts.isEmpty()) { + LOG.warn(HiveConf.ConfVars.HIVETEZJAVAOPTS + " will be ignored because " + + HiveConf.ConfVars.HIVETEZCONTAINERSIZE + " is not set!"); } - return javaOpts + " " + sb.toString(); + return logLevel + " " + MRHelpers.getJavaOptsForMRMapper(conf); } - return MRHelpers.getJavaOptsForMRMapper(conf); } private Vertex createVertex(JobConf conf, MergeJoinWork mergeJoinWork, LocalResource appJarLr, @@ -473,13 +480,9 @@ private Vertex createVertex(JobConf conf, MergeJoinWork mergeJoinWork, LocalReso if (mergeJoinWork.getMainWork() instanceof MapWork) { List mapWorkList = mergeJoinWork.getBaseWorkList(); MapWork mapWork = (MapWork) (mergeJoinWork.getMainWork()); - CommonMergeJoinOperator mergeJoinOp = mergeJoinWork.getMergeJoinOperator(); Vertex mergeVx = createVertex(conf, mapWork, appJarLr, additionalLr, fs, mrScratchDir, ctx, vertexType); - // grouping happens in execution phase. Setting the class to TezGroupedSplitsInputFormat - // here would cause pre-mature grouping which would be incorrect. - Class inputFormatClass = HiveInputFormat.class; conf.setClass("mapred.input.format.class", HiveInputFormat.class, InputFormat.class); // mapreduce.tez.input.initializer.serialize.event.payload should be set // to false when using this plug-in to avoid getting a serialized event at run-time. @@ -496,9 +499,11 @@ private Vertex createVertex(JobConf conf, MergeJoinWork mergeJoinWork, LocalReso VertexManagerPluginDescriptor desc = VertexManagerPluginDescriptor.create(CustomPartitionVertex.class.getName()); + // the +1 to the size is because of the main work. CustomVertexConfiguration vertexConf = new CustomVertexConfiguration(mergeJoinWork.getMergeJoinOperator().getConf() - .getNumBuckets(), vertexType, mergeJoinWork.getBigTableAlias()); + .getNumBuckets(), vertexType, mergeJoinWork.getBigTableAlias(), + mapWorkList.size() + 1); DataOutputBuffer dob = new DataOutputBuffer(); vertexConf.write(dob); byte[] userPayload = dob.getData(); @@ -538,6 +543,7 @@ private Vertex createVertex(JobConf conf, MapWork mapWork, DataSourceDescriptor dataSource; int numTasks = -1; + @SuppressWarnings("rawtypes") Class inputFormatClass = conf.getClass("mapred.input.format.class", InputFormat.class); @@ -595,7 +601,13 @@ private Vertex createVertex(JobConf conf, MapWork mapWork, .setCustomInitializerDescriptor(descriptor).build(); } else { // Not HiveInputFormat, or a custom VertexManager will take care of grouping splits - dataSource = MRInputLegacy.createConfigBuilder(conf, inputFormatClass).groupSplits(false).build(); + if (vertexHasCustomInput) { + dataSource = + MultiMRInput.createConfigBuilder(conf, inputFormatClass).groupSplits(false).build(); + } else { + dataSource = + MRInputLegacy.createConfigBuilder(conf, inputFormatClass).groupSplits(false).build(); + } } } else { // Setup client side split generation. @@ -640,6 +652,8 @@ private Vertex createVertex(JobConf conf, MapWork mapWork, private JobConf initializeVertexConf(JobConf baseConf, Context context, ReduceWork reduceWork) { JobConf conf = new JobConf(baseConf); + conf.set(Operator.CONTEXT_NAME_KEY, reduceWork.getName()); + // Is this required ? conf.set("mapred.reducer.class", ExecReducer.class.getName()); @@ -745,6 +759,7 @@ public PreWarmVertex createPreWarmVertex(TezConfiguration conf, * @throws LoginException if we are unable to figure user information * @throws IOException when any dfs operation fails. */ + @SuppressWarnings("deprecation") public Path getDefaultDestDir(Configuration conf) throws LoginException, IOException { UserGroupInformation ugi = ShimLoader.getHadoopShims().getUGIForConf(conf); String userName = ShimLoader.getHadoopShims().getShortUserName(ugi); @@ -857,6 +872,7 @@ public FileStatus getHiveJarDirectory(Configuration conf) throws IOException, Lo return fstatus; } + @SuppressWarnings("deprecation") public static FileStatus validateTargetDir(Path path, Configuration conf) throws IOException { FileSystem fs = path.getFileSystem(conf); FileStatus fstatus = null; @@ -971,7 +987,7 @@ public LocalResource localizeResource(Path src, Path dest, Configuration conf) public JobConf createConfiguration(HiveConf hiveConf) throws IOException { hiveConf.setBoolean("mapred.mapper.new-api", false); - JobConf conf = new JobConf(hiveConf); + JobConf conf = new JobConf(new TezConfiguration(hiveConf)); conf.set("mapred.output.committer.class", NullOutputCommitter.class.getName()); @@ -1033,6 +1049,7 @@ private JobConf initializeVertexConf(JobConf conf, Context context, MergeJoinWor * @param ctx This query's context * @return Vertex */ + @SuppressWarnings("deprecation") public Vertex createVertex(JobConf conf, BaseWork work, Path scratchDir, LocalResource appJarLr, List additionalLr, FileSystem fileSystem, Context ctx, boolean hasChildren, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DynamicPartitionPruner.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DynamicPartitionPruner.java index 7ba2ae1..696874e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DynamicPartitionPruner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DynamicPartitionPruner.java @@ -87,6 +87,8 @@ private final Object endOfEvents = new Object(); + private int totalEventCount = 0; + public DynamicPartitionPruner() { } @@ -114,7 +116,7 @@ public void prune(MapWork work, JobConf jobConf, InputInitializerContext context // synchronous event processing loop. Won't return until all events have // been processed. this.processEvents(); - this.prunePartitions(work); + this.prunePartitions(work, context); LOG.info("Ok to proceed."); } @@ -163,12 +165,22 @@ public void initialize(MapWork work, JobConf jobConf) throws SerDeException { } } - private void prunePartitions(MapWork work) throws HiveException { + private void prunePartitions(MapWork work, InputInitializerContext context) throws HiveException { + int expectedEvents = 0; for (String source : this.sourceInfoMap.keySet()) { for (SourceInfo si : this.sourceInfoMap.get(source)) { + int taskNum = context.getVertexNumTasks(source); + LOG.info("Expecting " + taskNum + " events for vertex " + source); + expectedEvents += taskNum; prunePartitionSingleSource(source, si, work); } } + + // sanity check. all tasks must submit events for us to succeed. + if (expectedEvents != totalEventCount) { + LOG.error("Expecting: " + expectedEvents + ", received: " + totalEventCount); + throw new HiveException("Incorrect event count in dynamic parition pruning"); + } } private void prunePartitionSingleSource(String source, SourceInfo si, MapWork work) @@ -396,7 +408,8 @@ public int read(byte[] bytes, int off, int len) throws IOException { public void addEvent(InputInitializerEvent event) { synchronized(sourcesWaitingForEvents) { if (sourcesWaitingForEvents.contains(event.getSourceVertexName())) { - queue.offer(event); + ++totalEventCount; + queue.offer(event); } } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java index c45479f..afe83d9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.ql.exec.tez; +import java.io.IOException; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -64,7 +66,6 @@ * making sure that splits from different partitions are only grouped if they * are of the same schema, format and serde */ -@SuppressWarnings("deprecation") public class HiveSplitGenerator extends InputInitializer { private static final Log LOG = LogFactory.getLog(HiveSplitGenerator.class); @@ -72,11 +73,17 @@ private static final SplitGrouper grouper = new SplitGrouper(); private final DynamicPartitionPruner pruner = new DynamicPartitionPruner(); private InputInitializerContext context; + private static Map, Map> cache = + new HashMap, Map>(); public HiveSplitGenerator(InputInitializerContext initializerContext) { super(initializerContext); } + public HiveSplitGenerator() { + this(null); + } + @Override public List initialize() throws Exception { InputInitializerContext rootInputContext = getContext(); @@ -150,58 +157,28 @@ public HiveSplitGenerator(InputInitializerContext initializerContext) { } - public static Multimap generateGroupedSplits(JobConf jobConf, + public Multimap generateGroupedSplits(JobConf jobConf, Configuration conf, InputSplit[] splits, float waves, int availableSlots) throws Exception { - return generateGroupedSplits(jobConf, conf, splits, waves, availableSlots, null); + return generateGroupedSplits(jobConf, conf, splits, waves, availableSlots, null, true); } - public static Multimap generateGroupedSplits(JobConf jobConf, - Configuration conf, InputSplit[] splits, float waves, int availableSlots, - String inputName) throws Exception { - - MapWork work = null; - if (inputName != null) { - work = (MapWork) Utilities.getMergeWork(jobConf, inputName); - // work can still be null if there is no merge work for this input - } - if (work == null) { - work = Utilities.getMapWork(jobConf); - } + public Multimap generateGroupedSplits(JobConf jobConf, + Configuration conf, InputSplit[] splits, float waves, int availableSlots, String inputName, + boolean groupAcrossFiles) throws Exception { + MapWork work = populateMapWork(jobConf, inputName); Multimap bucketSplitMultiMap = ArrayListMultimap. create(); - Class previousInputFormatClass = null; - String previousDeserializerClass = null; - Map, Map> cache = - new HashMap, Map>(); - int i = 0; - + InputSplit prevSplit = null; for (InputSplit s : splits) { // this is the bit where we make sure we don't group across partition // schema boundaries - - Path path = ((FileSplit) s).getPath(); - - PartitionDesc pd = - HiveFileFormatUtils.getPartitionDescFromPathRecursively(work.getPathToPartitionInfo(), - path, cache); - - String currentDeserializerClass = pd.getDeserializerClassName(); - Class currentInputFormatClass = pd.getInputFileFormatClass(); - - if ((currentInputFormatClass != previousInputFormatClass) - || (!currentDeserializerClass.equals(previousDeserializerClass))) { + if (schemaEvolved(s, prevSplit, groupAcrossFiles, work)) { ++i; - } - - previousInputFormatClass = currentInputFormatClass; - previousDeserializerClass = currentDeserializerClass; - - if (LOG.isDebugEnabled()) { - LOG.debug("Adding split " + path + " to src group " + i); + prevSplit = s; } bucketSplitMultiMap.put(i, s); } @@ -214,6 +191,54 @@ public HiveSplitGenerator(InputInitializerContext initializerContext) { return groupedSplits; } + private MapWork populateMapWork(JobConf jobConf, String inputName) { + MapWork work = null; + if (inputName != null) { + work = (MapWork) Utilities.getMergeWork(jobConf, inputName); + // work can still be null if there is no merge work for this input + } + if (work == null) { + work = Utilities.getMapWork(jobConf); + } + + return work; + } + + public boolean schemaEvolved(InputSplit s, InputSplit prevSplit, boolean groupAcrossFiles, + MapWork work) throws IOException { + boolean retval = false; + Path path = ((FileSplit) s).getPath(); + PartitionDesc pd = + HiveFileFormatUtils.getPartitionDescFromPathRecursively(work.getPathToPartitionInfo(), + path, cache); + String currentDeserializerClass = pd.getDeserializerClassName(); + Class currentInputFormatClass = pd.getInputFileFormatClass(); + + Class previousInputFormatClass = null; + String previousDeserializerClass = null; + if (prevSplit != null) { + Path prevPath = ((FileSplit) prevSplit).getPath(); + if (!groupAcrossFiles) { + return !path.equals(prevPath); + } + PartitionDesc prevPD = + HiveFileFormatUtils.getPartitionDescFromPathRecursively(work.getPathToPartitionInfo(), + prevPath, cache); + previousDeserializerClass = prevPD.getDeserializerClassName(); + previousInputFormatClass = prevPD.getInputFileFormatClass(); + } + + if ((currentInputFormatClass != previousInputFormatClass) + || (!currentDeserializerClass.equals(previousDeserializerClass))) { + retval = true; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Adding split " + path + " to src new group? " + retval); + } + return retval; + } + private List createEventList(boolean sendSerializedEvents, InputSplitInfoMem inputSplitInfo) { List events = Lists.newArrayListWithCapacity(inputSplitInfo.getNumTasks() + 1); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java index c77e081..bc7603e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java @@ -17,10 +17,10 @@ */ package org.apache.hadoop.hive.ql.exec.tez; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -44,15 +44,15 @@ import org.apache.hadoop.hive.ql.exec.tez.TezProcessor.TezKVOutputCollector; import org.apache.hadoop.hive.ql.exec.tez.tools.KeyValueInputMerger; import org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator; -import org.apache.hadoop.hive.ql.io.IOContext; import org.apache.hadoop.hive.ql.log.PerfLogger; import org.apache.hadoop.hive.ql.plan.MapWork; import org.apache.hadoop.hive.ql.plan.OperatorDesc; +import org.apache.hadoop.hive.serde2.Deserializer; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.mapred.JobConf; import org.apache.tez.mapreduce.input.MRInputLegacy; import org.apache.tez.mapreduce.input.MultiMRInput; import org.apache.tez.mapreduce.processor.MRTaskReporter; -import org.apache.tez.runtime.api.Input; import org.apache.tez.runtime.api.LogicalInput; import org.apache.tez.runtime.api.LogicalOutput; import org.apache.tez.runtime.api.ProcessorContext; @@ -73,6 +73,7 @@ private int position = 0; private boolean foundCachedMergeWork = false; MRInputLegacy legacyMRInput = null; + MultiMRInput mainWorkMultiMRInput = null; private ExecMapperContext execContext = null; private boolean abort = false; protected static final String MAP_PLAN_KEY = "__MAP_PLAN__"; @@ -129,12 +130,14 @@ void init(JobConf jconf, ProcessorContext processorContext, MRTaskReporter mrRep perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TEZ_INIT_OPERATORS); super.init(jconf, processorContext, mrReporter, inputs, outputs); - //Update JobConf using MRInput, info like filename comes via this + // Update JobConf using MRInput, info like filename comes via this legacyMRInput = getMRInput(inputs); - Configuration updatedConf = legacyMRInput.getConfigUpdates(); - if (updatedConf != null) { - for (Entry entry : updatedConf) { - jconf.set(entry.getKey(), entry.getValue()); + if (legacyMRInput != null) { + Configuration updatedConf = legacyMRInput.getConfigUpdates(); + if (updatedConf != null) { + for (Entry entry : updatedConf) { + jconf.set(entry.getKey(), entry.getValue()); + } } } @@ -158,8 +161,6 @@ void init(JobConf jconf, ProcessorContext processorContext, MRTaskReporter mrRep if (mergeWorkList != null) { MapOperator mergeMapOp = null; for (MapWork mergeMapWork : mergeWorkList) { - processorContext.waitForAnyInputReady(Collections.singletonList((Input) (inputs - .get(mergeMapWork.getName())))); if (mergeMapWork.getVectorMode()) { mergeMapOp = new VectorMapOperator(); } else { @@ -235,11 +236,17 @@ void init(JobConf jconf, ProcessorContext processorContext, MRTaskReporter mrRep } private void initializeMapRecordSources() throws Exception { + int size = mergeMapOpList.size() + 1; // the +1 is for the main map operator itself sources = new MapRecordSource[size]; - KeyValueReader reader = legacyMRInput.getReader(); position = mapOp.getConf().getTag(); sources[position] = new MapRecordSource(); + KeyValueReader reader = null; + if (mainWorkMultiMRInput != null) { + reader = getKeyValueReader(mainWorkMultiMRInput.getKeyValueReaders(), mapOp); + } else { + reader = legacyMRInput.getReader(); + } sources[position].init(jconf, mapOp, reader); for (MapOperator mapOp : mergeMapOpList) { int tag = mapOp.getConf().getTag(); @@ -248,13 +255,28 @@ private void initializeMapRecordSources() throws Exception { MultiMRInput multiMRInput = multiMRInputMap.get(inputName); Collection kvReaders = multiMRInput.getKeyValueReaders(); l4j.debug("There are " + kvReaders.size() + " key-value readers for input " + inputName); - List kvReaderList = new ArrayList(kvReaders); - reader = new KeyValueInputMerger(kvReaderList); + reader = getKeyValueReader(kvReaders, mapOp); sources[tag].init(jconf, mapOp, reader); } ((TezContext) MapredContext.get()).setRecordSources(sources); } + @SuppressWarnings("deprecation") + private KeyValueReader getKeyValueReader(Collection keyValueReaders, + MapOperator mapOp) + throws Exception { + List kvReaderList = new ArrayList(keyValueReaders); + // this sets up the map operator contexts correctly + mapOp.initializeContexts(); + Deserializer deserializer = mapOp.getCurrentDeserializer(); + KeyValueReader reader = + new KeyValueInputMerger(kvReaderList, deserializer, + new ObjectInspector[] { deserializer.getObjectInspector() }, mapOp + .getConf() + .getSortCols()); + return reader; + } + private DummyStoreOperator getJoinParentOp(Operator mergeMapOp) { for (Operator childOp : mergeMapOp.getChildOperators()) { if ((childOp.getChildOperators() == null) || (childOp.getChildOperators().isEmpty())) { @@ -269,11 +291,7 @@ private DummyStoreOperator getJoinParentOp(Operator merg @Override void run() throws Exception { - while (sources[position].pushRecord()) { - if (isLogInfoEnabled) { - logProgress(); - } - } + while (sources[position].pushRecord()) {} } @Override @@ -305,10 +323,7 @@ void close(){ } } - if (isLogInfoEnabled) { - logCloseInfo(); - } - ReportStats rps = new ReportStats(reporter); + ReportStats rps = new ReportStats(reporter, jconf); mapOp.preorderMap(rps); return; } catch (Exception e) { @@ -342,7 +357,17 @@ private MRInputLegacy getMRInput(Map inputs) throws Except multiMRInputMap.put(inp.getKey(), (MultiMRInput) inp.getValue()); } } - theMRInput.init(); + if (theMRInput != null) { + theMRInput.init(); + } else { + String alias = mapWork.getAliasToWork().keySet().iterator().next(); + if (inputs.get(alias) instanceof MultiMRInput) { + mainWorkMultiMRInput = (MultiMRInput) inputs.get(alias); + } else { + throw new IOException("Unexpected input type found: " + + inputs.get(alias).getClass().getCanonicalName()); + } + } return theMRInput; } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordSource.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordSource.java index 0419568..f7d2661 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordSource.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordSource.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hive.ql.exec.tez; import java.io.IOException; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.ql.exec.MapOperator; @@ -28,7 +27,6 @@ import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.util.StringUtils; -import org.apache.tez.mapreduce.input.MRInput; import org.apache.tez.runtime.library.api.KeyValueReader; /** @@ -45,7 +43,7 @@ private final boolean grouped = false; void init(JobConf jconf, MapOperator mapOp, KeyValueReader reader) throws IOException { - execContext = new ExecMapperContext(jconf); + execContext = mapOp.getExecContext(); this.mapOp = mapOp; this.reader = reader; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MergeFileRecordProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MergeFileRecordProcessor.java index d2d1962..2998ae7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MergeFileRecordProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MergeFileRecordProcessor.java @@ -17,10 +17,13 @@ */ package org.apache.hadoop.hive.ql.exec.tez; +import java.io.IOException; +import java.util.Map; +import java.util.Map.Entry; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.MapredContext; import org.apache.hadoop.hive.ql.exec.ObjectCacheFactory; import org.apache.hadoop.hive.ql.exec.Operator; @@ -41,10 +44,6 @@ import org.apache.tez.runtime.api.ProcessorContext; import org.apache.tez.runtime.library.api.KeyValueReader; -import java.io.IOException; -import java.util.Map; -import java.util.Map.Entry; - /** * Record processor for fast merging of files. */ @@ -93,17 +92,17 @@ void init(JobConf jconf, ProcessorContext processorContext, MapWork mapWork = (MapWork) cache.retrieve(MAP_PLAN_KEY); if (mapWork == null) { mapWork = Utilities.getMapWork(jconf); - if (mapWork instanceof MergeFileWork) { - mfWork = (MergeFileWork) mapWork; - } else { - throw new RuntimeException("MapWork should be an instance of" + - " MergeFileWork."); - } cache.cache(MAP_PLAN_KEY, mapWork); } else { Utilities.setMapWork(jconf, mapWork); } + if (mapWork instanceof MergeFileWork) { + mfWork = (MergeFileWork) mapWork; + } else { + throw new RuntimeException("MapWork should be an instance of MergeFileWork."); + } + String alias = mfWork.getAliasToWork().keySet().iterator().next(); mergeOp = mfWork.getAliasToWork().get(alias); LOG.info(mergeOp.dump(0)); @@ -157,10 +156,7 @@ void close() { } mergeOp.close(abort); - if (isLogInfoEnabled) { - logCloseInfo(); - } - ExecMapper.ReportStats rps = new ExecMapper.ReportStats(reporter); + ExecMapper.ReportStats rps = new ExecMapper.ReportStats(reporter, jconf); mergeOp.preorderMap(rps); } catch (Exception e) { if (!abort) { @@ -191,9 +187,6 @@ private boolean processRow(Object key, Object value) { row[0] = key; row[1] = value; mergeOp.processOp(row, 0); - if (isLogInfoEnabled) { - logProgress(); - } } } catch (Throwable e) { abort = true; @@ -211,6 +204,7 @@ private boolean processRow(Object key, Object value) { private MRInputLegacy getMRInput(Map inputs) throws Exception { // there should be only one MRInput MRInputLegacy theMRInput = null; + LOG.info("VDK: the inputs are: " + inputs); for (Entry inp : inputs.entrySet()) { if (inp.getValue() instanceof MRInputLegacy) { if (theMRInput != null) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/RecordProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/RecordProcessor.java index 372c54d..258557f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/RecordProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/RecordProcessor.java @@ -52,13 +52,10 @@ // used to log memory usage periodically - public static MemoryMXBean memoryMXBean; protected boolean isLogInfoEnabled = false; protected boolean isLogTraceEnabled = false; protected MRTaskReporter reporter; - private long numRows = 0; - private long nextUpdateCntr = 1; protected PerfLogger perfLogger = PerfLogger.getPerfLogger(); protected String CLASS_NAME = RecordProcessor.class.getName(); @@ -79,11 +76,6 @@ void init(JobConf jconf, ProcessorContext processorContext, MRTaskReporter mrRep this.outputs = outputs; this.processorContext = processorContext; - // Allocate the bean at the beginning - - memoryMXBean = ManagementFactory.getMemoryMXBean(); - - l4j.info("maximum memory = " + memoryMXBean.getHeapMemoryUsage().getMax()); - isLogInfoEnabled = l4j.isInfoEnabled(); isLogTraceEnabled = l4j.isTraceEnabled(); @@ -110,37 +102,6 @@ void init(JobConf jconf, ProcessorContext processorContext, MRTaskReporter mrRep abstract void close(); - /** - * Log information to be logged at the end - */ - protected void logCloseInfo() { - long used_memory = memoryMXBean.getHeapMemoryUsage().getUsed(); - l4j.info("TezProcessor: processed " + numRows + " rows/groups: used memory = " + used_memory); - } - - /** - * Log number of records processed and memory used after processing many records - */ - protected void logProgress() { - numRows++; - if (numRows == nextUpdateCntr) { - long used_memory = memoryMXBean.getHeapMemoryUsage().getUsed(); - l4j.info("TezProcessor: processing " + numRows + " rows/groups: used memory = " + used_memory); - nextUpdateCntr = getNextUpdateRecordCounter(numRows); - } - } - - private long getNextUpdateRecordCounter(long cntr) { - // A very simple counter to keep track of number of rows processed by the - // reducer. It dumps - // every 1 million times, and quickly before that - if (cntr >= 1000000) { - return cntr + 1000000; - } - - return 10 * cntr; - } - protected void createOutputMap() { Preconditions.checkState(outMap == null, "Outputs should only be setup once"); outMap = Maps.newHashMap(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java index 941f97c..8910af6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java @@ -101,7 +101,7 @@ void init(JobConf jconf, ProcessorContext processorContext, MRTaskReporter mrRep sources[tag] = new ReduceRecordSource(); sources[tag].init(jconf, reducer, redWork.getVectorMode(), keyTableDesc, valueTableDesc, reader, tag == position, (byte) tag, - redWork.getScratchColumnVectorTypes()); + redWork.getAllScratchColumnVectorTypeMaps()); ois[tag] = sources[tag].getObjectInspector(); } @@ -165,11 +165,7 @@ void run() throws Exception { } // run the operator pipeline - while (sources[position].pushRecord()) { - if (isLogInfoEnabled) { - logProgress(); - } - } + while (sources[position].pushRecord()) {} } /** @@ -208,7 +204,7 @@ void close(){ dummyOp.close(abort); } } - ReportStats rps = new ReportStats(reporter); + ReportStats rps = new ReportStats(reporter, jconf); reducer.preorderMap(rps); } catch (Exception e) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/SplitGrouper.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/SplitGrouper.java index 38d74d5..f9c80c2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/SplitGrouper.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/SplitGrouper.java @@ -102,9 +102,19 @@ // compute the total size per bucket long totalSize = 0; + boolean earlyExit = false; for (int bucketId : bucketSplitMap.keySet()) { long size = 0; for (InputSplit s : bucketSplitMap.get(bucketId)) { + // the incoming split may not be a file split when we are re-grouping TezGroupedSplits in + // the case of SMB join. So in this case, we can do an early exit by not doing the + // calculation for bucketSizeMap. Each bucket will assume it can fill availableSlots * waves + // (preset to 0.5) for SMB join. + if (!(s instanceof FileSplit)) { + bucketTaskMap.put(bucketId, (int) (availableSlots * waves)); + earlyExit = true; + continue; + } FileSplit fsplit = (FileSplit) s; size += fsplit.getLength(); totalSize += fsplit.getLength(); @@ -112,6 +122,10 @@ bucketSizeMap.put(bucketId, size); } + if (earlyExit) { + return bucketTaskMap; + } + // compute the number of tasks for (int bucketId : bucketSizeMap.keySet()) { int numEstimatedTasks = 0; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobExecHelper.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobExecHelper.java new file mode 100644 index 0000000..3eb954d --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobExecHelper.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.tez; + +import java.lang.reflect.Method; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * TezJobExecHelper is a utility to safely call Tez functionality from + * common code paths. It will check if tez is available/installed before + * invoking the code. + */ +public class TezJobExecHelper { + + private static final Log LOG = LogFactory.getLog(TezJobExecHelper.class.getName()); + + public static void killRunningJobs() { + try { + Class.forName("org.apache.tez.dag.api.DAG"); + + // we have tez installed + ClassLoader classLoader = TezJobExecHelper.class.getClassLoader(); + Method method = classLoader.loadClass("org.apache.hadoop.hive.ql.exec.tez.TezJobMonitor") + .getMethod("killRunningJobs"); + method.invoke(null, null); + } + catch (Exception e) { + // It is not available do nothing + LOG.debug("Could not stop tez dags: ", e); + } + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java index 48c6b6a..9bcdeed 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java @@ -68,12 +68,7 @@ @Override public void run() { for (DAGClient c: shutdownList) { - try { - System.err.println("Trying to shutdown DAG"); - c.tryKillDAG(); - } catch (Exception e) { - // ignore - } + TezJobMonitor.killRunningJobs(); } try { for (TezSessionState s: TezSessionState.getOpenSessions()) { @@ -212,6 +207,21 @@ public int monitorExecution(final DAGClient dagClient, HiveTxnManager txnMgr, return rc; } + /** + * killRunningJobs tries to terminate execution of all + * currently running tez queries. No guarantees, best effort only. + */ + public static void killRunningJobs() { + for (DAGClient c: shutdownList) { + try { + System.err.println("Trying to shutdown DAG"); + c.tryKillDAG(); + } catch (Exception e) { + // ignore + } + } + } + private String printStatus(Map progressMap, String lastReport, LogHelper console) { StringBuffer reportBuffer = new StringBuffer(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezProcessor.java index 42c7d37..1e528a9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezProcessor.java @@ -19,12 +19,9 @@ import java.io.IOException; import java.text.NumberFormat; -import java.util.Arrays; -import java.util.HashMap; +import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Map.Entry; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -33,11 +30,10 @@ import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.util.StringUtils; import org.apache.tez.common.TezUtils; -import org.apache.tez.mapreduce.input.MRInputLegacy; -import org.apache.tez.mapreduce.input.MultiMRInput; import org.apache.tez.mapreduce.processor.MRTaskReporter; import org.apache.tez.runtime.api.AbstractLogicalIOProcessor; import org.apache.tez.runtime.api.Event; +import org.apache.tez.runtime.api.Input; import org.apache.tez.runtime.api.LogicalInput; import org.apache.tez.runtime.api.LogicalOutput; import org.apache.tez.runtime.api.ProcessorContext; @@ -152,10 +148,13 @@ protected void initializeAndRunProcessor(Map inputs, // Start the actual Inputs. After MRInput initialization. for (Map.Entry inputEntry : inputs.entrySet()) { if (!cacheAccess.isInputCached(inputEntry.getKey())) { - LOG.info("Input: " + inputEntry.getKey() + " is not cached"); + LOG.info("Starting input " + inputEntry.getKey()); inputEntry.getValue().start(); + processorContext.waitForAnyInputReady(Collections.singletonList((Input) (inputEntry + .getValue()))); } else { - LOG.info("Input: " + inputEntry.getKey() + " is already cached. Skipping start"); + LOG.info("Input: " + inputEntry.getKey() + + " is already cached. Skipping start and wait for ready"); } } @@ -194,6 +193,7 @@ protected void initializeAndRunProcessor(Map inputs, * Must be initialized before it is used. * */ + @SuppressWarnings("rawtypes") static class TezKVOutputCollector implements OutputCollector { private KeyValueWriter writer; private final LogicalOutput output; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java index a4fd36d..e5fce14 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java @@ -141,7 +141,9 @@ private TezSessionState getSession(HiveConf conf, boolean doOpen, private TezSessionState getNewSessionState(HiveConf conf, String queueName, boolean doOpen) throws Exception { TezSessionState retTezSessionState = createSession(TezSessionState.makeSessionId()); - retTezSessionState.setQueueName(queueName); + if (queueName != null) { + conf.set("tez.queue.name", queueName); + } String what = "Created"; if (doOpen) { retTezSessionState.open(conf); @@ -221,29 +223,27 @@ private boolean canWorkWithSameSession(TezSessionState session, HiveConf conf) throw new HiveException(e); } - HiveConf existingConf = session.getConf(); - if (existingConf == null) { - return false; - } - + boolean doAsEnabled = conf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS); // either variables will never be null because a default value is returned in case of absence - if (existingConf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS) != - conf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS)) { + if (doAsEnabled != conf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS)) { return false; } if (!session.isDefault()) { - if (existingConf.get("tez.queue.name") == conf.get("tez.queue.name")) { - // both are null - return true; - } - if ((existingConf.get("tez.queue.name") == null)) { - // doesn't matter if the other conf is null or not. if it is null, above case catches it - return false; + String queueName = session.getQueueName(); + LOG.info("Current queue name is " + queueName + " incoming queue name is " + + conf.get("tez.queue.name")); + if (queueName == null) { + if (conf.get("tez.queue.name") != null) { + // queue names are different + return false; + } else { + return true; + } } - if (!existingConf.get("tez.queue.name").equals(conf.get("tez.queue.name"))) { - // handles the case of incoming conf having a null for tez.queue.name + if (!queueName.equals(conf.get("tez.queue.name"))) { + // the String.equals method handles the case of conf not having the queue name as well. return false; } } else { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java index 22ecac5..563fb49 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java @@ -67,13 +67,14 @@ private LocalResource appJarLr; private TezClient session; private String sessionId; - private DagUtils utils; + private final DagUtils utils; private String queueName; private boolean defaultQueue = false; private String user; private final Set additionalFilesNotFromConf = new HashSet(); private final Set localizedResources = new HashSet(); + private boolean doAsEnabled; private static List openSessions = Collections.synchronizedList(new LinkedList()); @@ -130,6 +131,8 @@ public void open(HiveConf conf) public void open(HiveConf conf, String[] additionalFiles) throws IOException, LoginException, IllegalArgumentException, URISyntaxException, TezException { this.conf = conf; + this.queueName = conf.get("tez.queue.name"); + this.doAsEnabled = conf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS); UserGroupInformation ugi; ugi = ShimLoader.getHadoopShims().getUGIForConf(conf); @@ -207,11 +210,6 @@ public void open(HiveConf conf, String[] additionalFiles) } catch(InterruptedException ie) { //ignore } - // In case we need to run some MR jobs, we'll run them under tez MR emulation. The session - // id is used for tez to reuse the current session rather than start a new one. - conf.set("mapreduce.framework.name", "yarn-tez"); - conf.set("mapreduce.tez.session.tokill-application-id", - session.getAppMasterApplicationId().toString()); openSessions.add(this); } @@ -397,4 +395,8 @@ public HiveConf getConf() { public String getUser() { return user; } + + public boolean getDoAsEnabled() { + return doAsEnabled; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java index 93e0fac..337f2f4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java @@ -47,6 +47,7 @@ import org.apache.hadoop.hive.ql.plan.api.StageType; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.tez.common.counters.CounterGroup; @@ -272,6 +273,7 @@ DAG build(JobConf conf, TezWork work, Path scratchDir, // the name of the dag is what is displayed in the AM/Job UI DAG dag = DAG.create(work.getName()); + dag.setCredentials(conf.getCredentials()); for (BaseWork w: ws) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/tools/KeyValueInputMerger.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/tools/KeyValueInputMerger.java index 516722d..c8e9606 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/tools/KeyValueInputMerger.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/tools/KeyValueInputMerger.java @@ -18,13 +18,23 @@ package org.apache.hadoop.hive.ql.exec.tez.tools; import java.io.IOException; +import java.util.ArrayList; import java.util.Comparator; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.PriorityQueue; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.io.BinaryComparable; +import org.apache.hadoop.hive.serde2.Deserializer; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; +import org.apache.hadoop.io.Writable; import org.apache.tez.runtime.library.api.KeyValueReader; /** @@ -34,16 +44,36 @@ * Uses a priority queue to pick the KeyValuesReader of the input that is next in * sort order. */ +@SuppressWarnings("deprecation") public class KeyValueInputMerger extends KeyValueReader { public static final Log l4j = LogFactory.getLog(KeyValueInputMerger.class); private PriorityQueue pQueue = null; private KeyValueReader nextKVReader = null; + private ObjectInspector[] inputObjInspectors = null; + private Deserializer deserializer = null; + private List structFields = null; + private List fieldOIs = null; + private final Map> kvReaderStandardObjMap = + new HashMap>(); - public KeyValueInputMerger(List multiMRInputs) throws Exception { + public KeyValueInputMerger(List multiMRInputs, Deserializer deserializer, + ObjectInspector[] inputObjInspectors, List sortCols) throws Exception { //get KeyValuesReaders from the LogicalInput and add them to priority queue int initialCapacity = multiMRInputs.size(); pQueue = new PriorityQueue(initialCapacity, new KVReaderComparator()); + this.inputObjInspectors = inputObjInspectors; + this.deserializer = deserializer; + fieldOIs = new ArrayList(); + structFields = new ArrayList(); + StructObjectInspector structOI = (StructObjectInspector) inputObjInspectors[0]; + for (String field : sortCols) { + StructField sf = structOI.getStructFieldRef(field); + structFields.add(sf); + ObjectInspector stdOI = + ObjectInspectorUtils.getStandardObjectInspector(sf.getFieldObjectInspector()); + fieldOIs.add(stdOI); + } l4j.info("Initialized the priority queue with multi mr inputs: " + multiMRInputs.size()); for (KeyValueReader input : multiMRInputs) { addToQueue(input); @@ -58,6 +88,7 @@ public KeyValueInputMerger(List multiMRInputs) throws Exception */ private void addToQueue(KeyValueReader kvReader) throws IOException { if (kvReader.next()) { + kvReaderStandardObjMap.remove(kvReader); pQueue.add(kvReader); } } @@ -93,12 +124,53 @@ public Object getCurrentValue() throws IOException { */ class KVReaderComparator implements Comparator { + @SuppressWarnings({ "unchecked" }) @Override public int compare(KeyValueReader kvReadr1, KeyValueReader kvReadr2) { try { - BinaryComparable key1 = (BinaryComparable) kvReadr1.getCurrentValue(); - BinaryComparable key2 = (BinaryComparable) kvReadr2.getCurrentValue(); - return key1.compareTo(key2); + ObjectInspector oi = inputObjInspectors[0]; + List row1, row2; + try { + if (kvReaderStandardObjMap.containsKey(kvReadr1)) { + row1 = kvReaderStandardObjMap.get(kvReadr1); + } else { + // we need to copy to standard object otherwise deserializer overwrites the values + row1 = + (List) ObjectInspectorUtils.copyToStandardObject( + deserializer.deserialize((Writable) kvReadr1.getCurrentValue()), oi, + ObjectInspectorCopyOption.WRITABLE); + kvReaderStandardObjMap.put(kvReadr1, row1); + } + + if (kvReaderStandardObjMap.containsKey(kvReadr2)) { + row2 = kvReaderStandardObjMap.get(kvReadr2); + } else { + row2 = + (List) ObjectInspectorUtils.copyToStandardObject( + deserializer.deserialize((Writable) kvReadr2.getCurrentValue()), oi, + ObjectInspectorCopyOption.WRITABLE); + kvReaderStandardObjMap.put(kvReadr2, row2); + } + } catch (SerDeException e) { + throw new IOException(e); + } + + StructObjectInspector structOI = (StructObjectInspector) oi; + int compare = 0; + int index = 0; + for (StructField sf : structFields) { + int pos = structOI.getAllStructFieldRefs().indexOf(sf); + Object key1 = row1.get(pos); + Object key2 = row2.get(pos); + ObjectInspector stdOI = fieldOIs.get(index); + compare = ObjectInspectorUtils.compare(key1, stdOI, key2, stdOI); + index++; + if (compare != 0) { + return compare; + } + } + + return compare; } catch (IOException e) { l4j.error("Caught exception while reading shuffle input", e); //die! diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractOperator.java index 1ddcbc6..7f4bb64 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractOperator.java @@ -18,85 +18,118 @@ package org.apache.hadoop.hive.ql.exec.vector; -import java.util.ArrayList; -import java.util.Arrays; import java.util.List; +import java.util.ArrayList; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; -import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorFactory; import org.apache.hadoop.hive.ql.exec.ExtractOperator; -import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.ExtractDesc; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.StructField; -import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; /** - * Vectorized extract operator implementation. Consumes rows and outputs a - * vectorized batch of subobjects. + * Vectorized extract operator implementation. **/ -public class VectorExtractOperator extends ExtractOperator { +public class VectorExtractOperator extends ExtractOperator implements VectorizationContextRegion { private static final long serialVersionUID = 1L; - private int keyColCount; - private int valueColCount; - - private transient VectorizedRowBatch outputBatch; - private transient int remainingColCount; + private List reduceTypeInfos; + + // Create a new outgoing vectorization context because we will project just the values. + private VectorizationContext vOutContext; + + private int[] projectedColumns; + private String removeValueDotPrefix(String columnName) { + return columnName.substring("VALUE.".length()); + } public VectorExtractOperator(VectorizationContext vContext, OperatorDesc conf) throws HiveException { this(); this.conf = (ExtractDesc) conf; + + List reduceColumnNames = vContext.getProjectionColumnNames(); + int reduceColCount = reduceColumnNames.size(); + + /* + * Create a new vectorization context as projection of just the values columns, but + * keep same output column manager must be inherited to track the scratch the columns. + */ + vOutContext = new VectorizationContext(vContext); + + // Set a fileKey with vectorization context. + vOutContext.setFileKey(vContext.getFileKey() + "/_EXTRACT_"); + + // Remove "VALUE." prefix from value columns and create a new projection + vOutContext.resetProjectionColumns(); + for (int i = 0; i < reduceColCount; i++) { + String columnName = reduceColumnNames.get(i); + if (columnName.startsWith("VALUE.")) { + vOutContext.addProjectionColumn(removeValueDotPrefix(columnName), i); + } + } } public VectorExtractOperator() { super(); } + /* + * Called by the Vectorizer class to pass the types from reduce shuffle. + */ + public void setReduceTypeInfos(List reduceTypeInfos) { + this.reduceTypeInfos = reduceTypeInfos; + } + @Override protected void initializeOp(Configuration hconf) throws HiveException { - StructObjectInspector structInputObjInspector = (StructObjectInspector) inputObjInspectors[0]; - List fields = structInputObjInspector.getAllStructFieldRefs(); - ArrayList ois = new ArrayList(); - ArrayList colNames = new ArrayList(); - for (int i = keyColCount; i < fields.size(); i++) { - StructField field = fields.get(i); - String fieldName = field.getFieldName(); - - // Remove "VALUE." prefix. - int dotIndex = fieldName.indexOf("."); - colNames.add(fieldName.substring(dotIndex + 1)); - ois.add(field.getFieldObjectInspector()); + // Create the projection of the values and the output object inspector + // for just the value without their "VALUE." prefix. + int projectionSize = vOutContext.getProjectedColumns().size(); + projectedColumns = new int[projectionSize]; + List columnNames = new ArrayList(); + List ois = new ArrayList(); + for (int i = 0; i < projectionSize; i++) { + int projectedIndex = vOutContext.getProjectedColumns().get(i); + projectedColumns[i] = projectedIndex; + String colName = vOutContext.getProjectionColumnNames().get(i); + columnNames.add(colName); + TypeInfo typeInfo = reduceTypeInfos.get(projectedIndex); + ObjectInspector oi = TypeInfoUtils + .getStandardWritableObjectInspectorFromTypeInfo(typeInfo); + ois.add(oi); } - outputObjInspector = ObjectInspectorFactory - .getStandardStructObjectInspector(colNames, ois); - remainingColCount = fields.size() - keyColCount; - outputBatch = new VectorizedRowBatch(remainingColCount); + outputObjInspector = ObjectInspectorFactory. + getStandardStructObjectInspector(columnNames, ois); initializeChildren(hconf); } - public void setKeyAndValueColCounts(int keyColCount, int valueColCount) { - this.keyColCount = keyColCount; - this.valueColCount = valueColCount; - } @Override // Remove the key columns and forward the values (and scratch columns). public void processOp(Object row, int tag) throws HiveException { - VectorizedRowBatch inputBatch = (VectorizedRowBatch) row; + VectorizedRowBatch vrg = (VectorizedRowBatch) row; - // Copy references to the input columns array starting after the keys... - for (int i = 0; i < remainingColCount; i++) { - outputBatch.cols[i] = inputBatch.cols[keyColCount + i]; - } - outputBatch.size = inputBatch.size; + int[] originalProjections = vrg.projectedColumns; + int originalProjectionSize = vrg.projectionSize; - forward(outputBatch, outputObjInspector); + // Temporarily substitute our projection. + vrg.projectionSize = projectedColumns.length; + vrg.projectedColumns = projectedColumns; + + forward(vrg, null); + + // Revert the projected columns back, because vrg will be re-used. + vrg.projectionSize = originalProjectionSize; + vrg.projectedColumns = originalProjections; + } + + @Override + public VectorizationContext getOuputVectorizationContext() { + return vOutContext; } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFileSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFileSinkOperator.java index ea32f33..858604c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFileSinkOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFileSinkOperator.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.FileSinkDesc; import org.apache.hadoop.hive.ql.plan.OperatorDesc; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; /** @@ -50,10 +51,22 @@ public VectorFileSinkOperator() { @Override protected void initializeOp(Configuration hconf) throws HiveException { - super.initializeOp(hconf); - valueWriters = VectorExpressionWriterFactory.getExpressionWriters( - (StructObjectInspector) inputObjInspectors[0]); + // We need a input object inspector that is for the row we will extract out of the + // vectorized row batch, not for example, an original inspector for an ORC table, etc. + VectorExpressionWriterFactory.processVectorInspector( + (StructObjectInspector) inputObjInspectors[0], + new VectorExpressionWriterFactory.SingleOIDClosure() { + @Override + public void assign(VectorExpressionWriter[] writers, + ObjectInspector objectInspector) { + valueWriters = writers; + inputObjInspectors[0] = objectInspector; + } + }); singleRow = new Object[valueWriters.length]; + + // Call FileSinkOperator with new input inspector. + super.initializeOp(hconf); } @Override diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFilterOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFilterOperator.java index 907ace1..e0f9546 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFilterOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFilterOperator.java @@ -60,8 +60,6 @@ protected void initializeOp(Configuration hconf) throws HiveException { try { heartbeatInterval = HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVESENDHEARTBEAT); - statsMap.put(Counter.FILTERED, filtered_count); - statsMap.put(Counter.PASSED, passed_count); } catch (Throwable e) { throw new HiveException(e); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java index 6274cb6..856ff20 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java @@ -32,11 +32,8 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; -import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorFactory; import org.apache.hadoop.hive.ql.exec.GroupByOperator; import org.apache.hadoop.hive.ql.exec.KeyWrapper; -import org.apache.hadoop.hive.ql.exec.KeyWrapperFactory; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriter; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriterFactory; @@ -46,12 +43,9 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.GroupByDesc; import org.apache.hadoop.hive.ql.plan.OperatorDesc; -import org.apache.hadoop.hive.ql.plan.api.OperatorType; import org.apache.hadoop.hive.ql.util.JavaDataModel; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.io.DataOutputBuffer; @@ -760,13 +754,7 @@ public VectorGroupByOperator(VectorizationContext vContext, OperatorDesc conf) isVectorOutput = desc.getVectorDesc().isVectorOutput(); - List outColNames = desc.getOutputColumnNames(); - Map mapOutCols = new HashMap(outColNames.size()); - int outColIndex = 0; - for(String outCol: outColNames) { - mapOutCols.put(outCol, outColIndex++); - } - vOutContext = new VectorizationContext(mapOutCols, outColIndex); + vOutContext = new VectorizationContext(desc.getOutputColumnNames()); vOutContext.setFileKey(vContext.getFileKey() + "/_GROUPBY_"); fileKey = vOutContext.getFileKey(); } @@ -811,7 +799,7 @@ protected void initializeOp(Configuration hconf) throws HiveException { vrbCtx.init(hconf, fileKey, (StructObjectInspector) outputObjInspector); outputBatch = vrbCtx.createVectorizedRowBatch(); vectorColumnAssign = VectorColumnAssignFactory.buildAssigners( - outputBatch, outputObjInspector, vOutContext.getColumnMap(), conf.getOutputColumnNames()); + outputBatch, outputObjInspector, vOutContext.getProjectionColumnMap(), conf.getOutputColumnNames()); } } catch (HiveException he) { @@ -851,11 +839,19 @@ private void changeToUnsortedStreamingMode() throws HiveException { @Override public void startGroup() throws HiveException { processingMode.startGroup(); + + // We do not call startGroup on operators below because we are batching rows in + // an output batch and the semantics will not work. + // super.startGroup(); } @Override public void endGroup() throws HiveException { processingMode.endGroup(); + + // We do not call endGroup on operators below because we are batching rows in + // an output batch and the semantics will not work. + // super.endGroup(); } @Override diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupKeyHelper.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupKeyHelper.java index 3c4f6cf..96c4498 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupKeyHelper.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupKeyHelper.java @@ -112,7 +112,10 @@ public void copyGroupKey(VectorizedRowBatch inputBatch, VectorizedRowBatch outpu DecimalColumnVector inputColumnVector = (DecimalColumnVector) inputBatch.cols[keyIndex]; DecimalColumnVector outputColumnVector = (DecimalColumnVector) outputBatch.cols[keyIndex]; if (inputColumnVector.noNulls || !inputColumnVector.isNull[0]) { - outputColumnVector.vector[outputBatch.size] = inputColumnVector.vector[0]; + + // Since we store references to Decimal128 instances, we must use the update method instead + // of plain assignment. + outputColumnVector.vector[outputBatch.size].update(inputColumnVector.vector[0]); } else { outputColumnVector.noNulls = false; outputColumnVector.isNull[outputBatch.size] = true; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java index 30e36ac..5fda5dd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java @@ -28,11 +28,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; import org.apache.hadoop.hive.ql.exec.MapJoinOperator; -import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.exec.Utilities; -import org.apache.hadoop.hive.ql.exec.persistence.MapJoinKeyObject; -import org.apache.hadoop.hive.ql.exec.persistence.MapJoinKey; -import org.apache.hadoop.hive.ql.exec.persistence.MapJoinKey; import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer.ReusableGetAdaptor; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriter; @@ -41,8 +37,6 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.MapJoinDesc; import org.apache.hadoop.hive.ql.plan.OperatorDesc; -import org.apache.hadoop.hive.ql.plan.api.OperatorType; -import org.apache.hadoop.hive.serde2.ByteStream.Output; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; @@ -116,16 +110,8 @@ public VectorMapJoinOperator (VectorizationContext vContext, OperatorDesc conf) Map> exprs = desc.getExprs(); bigTableValueExpressions = vContext.getVectorExpressions(exprs.get(posBigTable)); - List outColNames = desc.getOutputColumnNames(); - - Map mapOutCols = new HashMap(outColNames.size()); - - int outColIndex = 0; - for(String outCol: outColNames) { - mapOutCols.put(outCol, outColIndex++); - } - - vOutContext = new VectorizationContext(mapOutCols, outColIndex); + // We are making a new output vectorized row batch. + vOutContext = new VectorizationContext(desc.getOutputColumnNames()); vOutContext.setFileKey(vContext.getFileKey() + "/MAP_JOIN_" + desc.getBigTableAlias()); this.fileKey = vOutContext.getFileKey(); } @@ -207,7 +193,7 @@ protected void internalForward(Object row, ObjectInspector outputOI) throws Hive Object[] values = (Object[]) row; VectorColumnAssign[] vcas = outputVectorAssigners.get(outputOI); if (null == vcas) { - Map> allColumnMaps = Utilities.getScratchColumnMap(hconf); + Map> allColumnMaps = Utilities.getAllColumnVectorMaps(hconf); Map columnMap = allColumnMaps.get(fileKey); vcas = VectorColumnAssignFactory.buildAssigners( outputBatch, outputOI, columnMap, conf.getOutputColumnNames()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java index 311f6d6..aa0d5a5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java @@ -40,7 +40,14 @@ public void process(Writable value) throws HiveException { // The row has been converted to comply with table schema, irrespective of partition schema. // So, use tblOI (and not partOI) for forwarding try { - forward(value, current.getRowObjectInspector()); + int childrenDone = 0; + for (MapOpCtx current : currentCtxs) { + if (!current.forward(value)) { + childrenDone++; + } + } + + rowsForwarded(childrenDone, ((VectorizedRowBatch)value).size); } catch (Exception e) { throw new HiveException("Hive Runtime Error while processing row ", e); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSMBMapJoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSMBMapJoinOperator.java index 0591531..22ffb6e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSMBMapJoinOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSMBMapJoinOperator.java @@ -27,16 +27,13 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; -import org.apache.hadoop.hive.ql.exec.JoinUtil; import org.apache.hadoop.hive.ql.exec.SMBMapJoinOperator; import org.apache.hadoop.hive.ql.exec.Utilities; -import org.apache.hadoop.hive.ql.exec.persistence.MapJoinKeyObject; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriter; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriterFactory; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; -import org.apache.hadoop.hive.ql.plan.GroupByDesc; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.SMBJoinDesc; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; @@ -116,19 +113,9 @@ public VectorSMBMapJoinOperator(VectorizationContext vContext, OperatorDesc conf Map> exprs = desc.getExprs(); bigTableValueExpressions = vContext.getVectorExpressions(exprs.get(posBigTable)); - - // Vectorized join operators need to create a new vectorization region for child operators. - - List outColNames = desc.getOutputColumnNames(); - - Map mapOutCols = new HashMap(outColNames.size()); - - int outColIndex = 0; - for(String outCol: outColNames) { - mapOutCols.put(outCol, outColIndex++); - } - vOutContext = new VectorizationContext(mapOutCols, outColIndex); + // We are making a new output vectorized row batch. + vOutContext = new VectorizationContext(desc.getOutputColumnNames()); vOutContext.setFileKey(vContext.getFileKey() + "/SMB_JOIN_" + desc.getBigTableAlias()); this.fileKey = vOutContext.getFileKey(); } @@ -285,7 +272,7 @@ protected void internalForward(Object row, ObjectInspector outputOI) throws Hive Object[] values = (Object[]) row; VectorColumnAssign[] vcas = outputVectorAssigners.get(outputOI); if (null == vcas) { - Map> allColumnMaps = Utilities.getScratchColumnMap(hconf); + Map> allColumnMaps = Utilities.getAllColumnVectorMaps(hconf); Map columnMap = allColumnMaps.get(fileKey); vcas = VectorColumnAssignFactory.buildAssigners( outputBatch, outputOI, columnMap, conf.getOutputColumnNames()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java index a65e594..458dc5a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java @@ -31,7 +31,6 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.SelectDesc; -import org.apache.hadoop.hive.ql.plan.api.OperatorType; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; @@ -63,20 +62,20 @@ public VectorSelectOperator(VectorizationContext vContext, OperatorDesc conf) } /** - * Create a new vectorization context to update the column map but same output column manager - * must be inherited to track the scratch the columns. + * Create a new vectorization context to create a new projection, but keep + * same output column manager must be inherited to track the scratch the columns. */ vOutContext = new VectorizationContext(vContext); // Set a fileKey, although this operator doesn't use it. vOutContext.setFileKey(vContext.getFileKey() + "/_SELECT_"); - // Update column map - vOutContext.getColumnMap().clear(); + vOutContext.resetProjectionColumns(); for (int i=0; i < colList.size(); ++i) { String columnName = this.conf.getOutputColumnNames().get(i); VectorExpression ve = vExpressions[i]; - vOutContext.addToColumnMap(columnName, ve.getOutputColumn()); + vOutContext.addProjectionColumn(columnName, + ve.getOutputColumn()); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java index fdd43e7..42ad37d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java @@ -23,11 +23,13 @@ import java.sql.Timestamp; import java.util.ArrayList; import java.util.Arrays; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeMap; import java.util.regex.Pattern; import org.apache.commons.logging.Log; @@ -123,9 +125,98 @@ VectorExpressionDescriptor vMap; + private List projectedColumns; + private List projectionColumnNames; + private Map projectionColumnMap; + //columnName to column position map - private final Map columnMap; - private final int firstOutputColumnIndex; + // private final Map columnMap; + private int firstOutputColumnIndex; + + // Convenient constructor for initial batch creation takes + // a list of columns names and maps them to 0..n-1 indices. + public VectorizationContext(List initialColumnNames) { + this.projectionColumnNames = initialColumnNames; + + projectedColumns = new ArrayList(); + projectionColumnMap = new HashMap(); + for (int i = 0; i < this.projectionColumnNames.size(); i++) { + projectedColumns.add(i); + projectionColumnMap.put(projectionColumnNames.get(i), i); + } + int firstOutputColumnIndex = projectedColumns.size(); + this.ocm = new OutputColumnManager(firstOutputColumnIndex); + this.firstOutputColumnIndex = firstOutputColumnIndex; + vMap = new VectorExpressionDescriptor(); + } + + // Constructor to with the individual addInitialColumn method + // followed by a call to finishedAddingInitialColumns. + public VectorizationContext() { + projectedColumns = new ArrayList(); + projectionColumnNames = new ArrayList(); + projectionColumnMap = new HashMap(); + this.ocm = new OutputColumnManager(0); + this.firstOutputColumnIndex = 0; + vMap = new VectorExpressionDescriptor(); + } + + // Constructor useful making a projection vectorization context. + // Use with resetProjectionColumns and addProjectionColumn. + // Keeps existing output column map, etc. + public VectorizationContext(VectorizationContext vContext) { + this.projectedColumns = new ArrayList(); + this.projectionColumnNames = new ArrayList(); + this.projectionColumnMap = new HashMap(); + + this.ocm = vContext.ocm; + this.firstOutputColumnIndex = vContext.firstOutputColumnIndex; + vMap = new VectorExpressionDescriptor(); + } + + // Add an initial column to a vectorization context when + // a vectorized row batch is being created. + public void addInitialColumn(String columnName) { + int index = projectedColumns.size(); + projectedColumns.add(index); + projectionColumnNames.add(columnName); + projectionColumnMap.put(columnName, index); + } + + // Finishes the vectorization context after all the initial + // columns have been added. + public void finishedAddingInitialColumns() { + int firstOutputColumnIndex = projectedColumns.size(); + this.ocm = new OutputColumnManager(firstOutputColumnIndex); + this.firstOutputColumnIndex = firstOutputColumnIndex; + } + + // Empties the projection columns. + public void resetProjectionColumns() { + projectedColumns = new ArrayList(); + projectionColumnNames = new ArrayList(); + projectionColumnMap = new HashMap(); + } + + // Add a projection column to a projection vectorization context. + public void addProjectionColumn(String columnName, int vectorBatchColIndex) { + projectedColumns.add(vectorBatchColIndex); + projectionColumnNames.add(columnName); + projectionColumnMap.put(columnName, vectorBatchColIndex); + } + + public List getProjectedColumns() { + return projectedColumns; + } + + public List getProjectionColumnNames() { + return projectionColumnNames; + } + + public Map getProjectionColumnMap() { + return projectionColumnMap; + } + public static final Pattern decimalTypePattern = Pattern.compile("decimal.*", Pattern.CASE_INSENSITIVE); @@ -140,7 +231,7 @@ Pattern.CASE_INSENSITIVE); //Map column number to type - private final OutputColumnManager ocm; + private OutputColumnManager ocm; // File key is used by operators to retrieve the scratch vectors // from mapWork at runtime. The operators that modify the structure of @@ -170,27 +261,6 @@ castExpressionUdfs.add(UDFToShort.class); } - public VectorizationContext(Map columnMap, - int initialOutputCol) { - this.columnMap = columnMap; - this.ocm = new OutputColumnManager(initialOutputCol); - this.firstOutputColumnIndex = initialOutputCol; - vMap = new VectorExpressionDescriptor(); - } - - /** - * This constructor inherits the OutputColumnManger and from - * the 'parent' constructor, therefore this should be used only by operators - * that don't create a new vectorized row batch. This should be used only by - * operators that want to modify the columnName map without changing the row batch. - */ - public VectorizationContext(VectorizationContext parent) { - this.columnMap = new HashMap(parent.columnMap); - this.ocm = parent.ocm; - this.firstOutputColumnIndex = parent.firstOutputColumnIndex; - vMap = new VectorExpressionDescriptor(); - } - public String getFileKey() { return fileKey; } @@ -199,16 +269,19 @@ public void setFileKey(String fileKey) { this.fileKey = fileKey; } - protected int getInputColumnIndex(String name) { - if (!columnMap.containsKey(name)) { - LOG.error(String.format("The column %s is not in the vectorization context column map %s.", - name, columnMap.toString())); + protected int getInputColumnIndex(String name) throws HiveException { + if (name == null) { + throw new HiveException("Null column name"); + } + if (!projectionColumnMap.containsKey(name)) { + throw new HiveException(String.format("The column %s is not in the vectorization context column map %s.", + name, projectionColumnMap.toString())); } - return columnMap.get(name); + return projectionColumnMap.get(name); } protected int getInputColumnIndex(ExprNodeColumnDesc colExpr) { - return columnMap.get(colExpr.getColumn()); + return projectionColumnMap.get(colExpr.getColumn()); } private static class OutputColumnManager { @@ -280,7 +353,7 @@ void freeOutputColumn(int index) { } private VectorExpression getColumnVectorExpression(ExprNodeColumnDesc - exprDesc, Mode mode) { + exprDesc, Mode mode) throws HiveException { int columnNum = getInputColumnIndex(exprDesc.getColumn()); VectorExpression expr = null; switch (mode) { @@ -1988,7 +2061,7 @@ public VectorAggregateExpression getAggregatorExpression(AggregationDesc desc, b "\" for type: \"" + inputType.name() + " (reduce-side = " + isReduce + ")"); } - public Map getOutputColumnTypeMap() { + public Map getScratchColumnTypeMap() { Map map = new HashMap(); for (int i = 0; i < ocm.outputColCount; i++) { String type = ocm.outputColumnsTypes[i]; @@ -1997,15 +2070,26 @@ public VectorAggregateExpression getAggregatorExpression(AggregationDesc desc, b return map; } - public Map getColumnMap() { - return columnMap; - } + public String toString() { + StringBuilder sb = new StringBuilder(32); + sb.append("Context key ").append(getFileKey()).append(", "); + + Comparator comparerInteger = new Comparator() { + @Override + public int compare(Integer o1, Integer o2) { + return o1.compareTo(o2); + }}; - public void addToColumnMap(String columnName, int outputColumn) throws HiveException { - if (columnMap.containsKey(columnName) && (columnMap.get(columnName) != outputColumn)) { - throw new HiveException(String.format("Column %s is already mapped to %d. Cannot remap to %d.", - columnName, columnMap.get(columnName), outputColumn)); + Map sortedColumnMap = new TreeMap(comparerInteger); + for (Map.Entry entry : projectionColumnMap.entrySet()) { + sortedColumnMap.put(entry.getValue(), entry.getKey()); } - columnMap.put(columnName, outputColumn); + sb.append("sortedProjectionColumnMap ").append(sortedColumnMap).append(", "); + + Map sortedScratchColumnTypeMap = new TreeMap(comparerInteger); + sortedScratchColumnTypeMap.putAll(getScratchColumnTypeMap()); + sb.append("sortedScratchColumnTypeMap ").append(sortedScratchColumnTypeMap); + + return sb.toString(); } - } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java index 1e24710..7b9c0a7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java @@ -23,6 +23,8 @@ import java.util.LinkedList; import java.util.List; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.common.type.HiveChar; import org.apache.hadoop.hive.common.type.HiveVarchar; import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr; @@ -50,6 +52,7 @@ import org.apache.hadoop.io.Text; public class VectorizedBatchUtil { + private static final Log LOG = LogFactory.getLog(VectorizedBatchUtil.class); /** * Sets the IsNull value for ColumnVector at specified index @@ -232,169 +235,237 @@ public static void addRowToBatchFrom(Object row, StructObjectInspector oi, final int off = colOffset; // Iterate thru the cols and load the batch for (int i = 0; i < fieldRefs.size(); i++) { - Object fieldData = oi.getStructFieldData(row, fieldRefs.get(i)); - ObjectInspector foi = fieldRefs.get(i).getFieldObjectInspector(); + setVector(row, oi, fieldRefs, batch, buffer, rowIndex, i, off); + } + } - // Vectorization only supports PRIMITIVE data types. Assert the same - assert (foi.getCategory() == Category.PRIMITIVE); + /** + * Iterates thru all the columns in a given row and populates the batch + * from a given offset + * + * @param row Deserialized row object + * @param oi Object insepector for that row + * @param rowIndex index to which the row should be added to batch + * @param batch Vectorized batch to which the row is added at rowIndex + * @param context context object for this vectorized batch + * @param buffer + * @throws HiveException + */ + public static void acidAddRowToBatch(Object row, + StructObjectInspector oi, + int rowIndex, + VectorizedRowBatch batch, + VectorizedRowBatchCtx context, + DataOutputBuffer buffer) throws HiveException { + List fieldRefs = oi.getAllStructFieldRefs(); + // Iterate thru the cols and load the batch + for (int i = 0; i < fieldRefs.size(); i++) { + if (batch.cols[i] == null) { + // This means the column was not included in the projection from the underlying read + continue; + } + if (context.isPartitionCol(i)) { + // The value will have already been set before we're called, so don't overwrite it + continue; + } + setVector(row, oi, fieldRefs, batch, buffer, rowIndex, i, 0); + } + } - // Get writable object - PrimitiveObjectInspector poi = (PrimitiveObjectInspector) foi; - Object writableCol = poi.getPrimitiveWritableObject(fieldData); + private static void setVector(Object row, + StructObjectInspector oi, + List fieldRefs, + VectorizedRowBatch batch, + DataOutputBuffer buffer, + int rowIndex, + int colIndex, + int offset) throws HiveException { - // NOTE: The default value for null fields in vectorization is 1 for int types, NaN for - // float/double. String types have no default value for null. - switch (poi.getPrimitiveCategory()) { - case BOOLEAN: { - LongColumnVector lcv = (LongColumnVector) batch.cols[off + i]; - if (writableCol != null) { - lcv.vector[rowIndex] = ((BooleanWritable) writableCol).get() ? 1 : 0; - lcv.isNull[rowIndex] = false; - } else { - lcv.vector[rowIndex] = 1; - setNullColIsNullValue(lcv, rowIndex); - } - } - break; - case BYTE: { - LongColumnVector lcv = (LongColumnVector) batch.cols[off + i]; - if (writableCol != null) { - lcv.vector[rowIndex] = ((ByteWritable) writableCol).get(); - lcv.isNull[rowIndex] = false; - } else { - lcv.vector[rowIndex] = 1; - setNullColIsNullValue(lcv, rowIndex); - } + Object fieldData = oi.getStructFieldData(row, fieldRefs.get(colIndex)); + ObjectInspector foi = fieldRefs.get(colIndex).getFieldObjectInspector(); + + // Vectorization only supports PRIMITIVE data types. Assert the same + assert (foi.getCategory() == Category.PRIMITIVE); + + // Get writable object + PrimitiveObjectInspector poi = (PrimitiveObjectInspector) foi; + Object writableCol = poi.getPrimitiveWritableObject(fieldData); + + // NOTE: The default value for null fields in vectorization is 1 for int types, NaN for + // float/double. String types have no default value for null. + switch (poi.getPrimitiveCategory()) { + case BOOLEAN: { + LongColumnVector lcv = (LongColumnVector) batch.cols[offset + colIndex]; + if (writableCol != null) { + lcv.vector[rowIndex] = ((BooleanWritable) writableCol).get() ? 1 : 0; + lcv.isNull[rowIndex] = false; + } else { + lcv.vector[rowIndex] = 1; + setNullColIsNullValue(lcv, rowIndex); } - break; - case SHORT: { - LongColumnVector lcv = (LongColumnVector) batch.cols[off + i]; - if (writableCol != null) { - lcv.vector[rowIndex] = ((ShortWritable) writableCol).get(); - lcv.isNull[rowIndex] = false; - } else { - lcv.vector[rowIndex] = 1; - setNullColIsNullValue(lcv, rowIndex); - } + } + break; + case BYTE: { + LongColumnVector lcv = (LongColumnVector) batch.cols[offset + colIndex]; + if (writableCol != null) { + lcv.vector[rowIndex] = ((ByteWritable) writableCol).get(); + lcv.isNull[rowIndex] = false; + } else { + lcv.vector[rowIndex] = 1; + setNullColIsNullValue(lcv, rowIndex); } - break; - case INT: { - LongColumnVector lcv = (LongColumnVector) batch.cols[off + i]; - if (writableCol != null) { - lcv.vector[rowIndex] = ((IntWritable) writableCol).get(); - lcv.isNull[rowIndex] = false; - } else { - lcv.vector[rowIndex] = 1; - setNullColIsNullValue(lcv, rowIndex); - } + } + break; + case SHORT: { + LongColumnVector lcv = (LongColumnVector) batch.cols[offset + colIndex]; + if (writableCol != null) { + lcv.vector[rowIndex] = ((ShortWritable) writableCol).get(); + lcv.isNull[rowIndex] = false; + } else { + lcv.vector[rowIndex] = 1; + setNullColIsNullValue(lcv, rowIndex); } - break; - case LONG: { - LongColumnVector lcv = (LongColumnVector) batch.cols[off + i]; - if (writableCol != null) { - lcv.vector[rowIndex] = ((LongWritable) writableCol).get(); - lcv.isNull[rowIndex] = false; - } else { - lcv.vector[rowIndex] = 1; - setNullColIsNullValue(lcv, rowIndex); - } + } + break; + case INT: { + LongColumnVector lcv = (LongColumnVector) batch.cols[offset + colIndex]; + if (writableCol != null) { + lcv.vector[rowIndex] = ((IntWritable) writableCol).get(); + lcv.isNull[rowIndex] = false; + } else { + lcv.vector[rowIndex] = 1; + setNullColIsNullValue(lcv, rowIndex); } - break; - case DATE: { - LongColumnVector lcv = (LongColumnVector) batch.cols[off + i]; - if (writableCol != null) { - lcv.vector[rowIndex] = ((DateWritable) writableCol).getDays(); - lcv.isNull[rowIndex] = false; - } else { - lcv.vector[rowIndex] = 1; - setNullColIsNullValue(lcv, rowIndex); - } + } + break; + case LONG: { + LongColumnVector lcv = (LongColumnVector) batch.cols[offset + colIndex]; + if (writableCol != null) { + lcv.vector[rowIndex] = ((LongWritable) writableCol).get(); + lcv.isNull[rowIndex] = false; + } else { + lcv.vector[rowIndex] = 1; + setNullColIsNullValue(lcv, rowIndex); } - break; - case FLOAT: { - DoubleColumnVector dcv = (DoubleColumnVector) batch.cols[off + i]; - if (writableCol != null) { - dcv.vector[rowIndex] = ((FloatWritable) writableCol).get(); - dcv.isNull[rowIndex] = false; - } else { - dcv.vector[rowIndex] = Double.NaN; - setNullColIsNullValue(dcv, rowIndex); - } + } + break; + case DATE: { + LongColumnVector lcv = (LongColumnVector) batch.cols[offset + colIndex]; + if (writableCol != null) { + lcv.vector[rowIndex] = ((DateWritable) writableCol).getDays(); + lcv.isNull[rowIndex] = false; + } else { + lcv.vector[rowIndex] = 1; + setNullColIsNullValue(lcv, rowIndex); } - break; - case DOUBLE: { - DoubleColumnVector dcv = (DoubleColumnVector) batch.cols[off + i]; - if (writableCol != null) { - dcv.vector[rowIndex] = ((DoubleWritable) writableCol).get(); - dcv.isNull[rowIndex] = false; - } else { - dcv.vector[rowIndex] = Double.NaN; - setNullColIsNullValue(dcv, rowIndex); - } + } + break; + case FLOAT: { + DoubleColumnVector dcv = (DoubleColumnVector) batch.cols[offset + colIndex]; + if (writableCol != null) { + dcv.vector[rowIndex] = ((FloatWritable) writableCol).get(); + dcv.isNull[rowIndex] = false; + } else { + dcv.vector[rowIndex] = Double.NaN; + setNullColIsNullValue(dcv, rowIndex); } - break; - case TIMESTAMP: { - LongColumnVector lcv = (LongColumnVector) batch.cols[off + i]; - if (writableCol != null) { - Timestamp t = ((TimestampWritable) writableCol).getTimestamp(); - lcv.vector[rowIndex] = TimestampUtils.getTimeNanoSec(t); - lcv.isNull[rowIndex] = false; - } else { - lcv.vector[rowIndex] = 1; - setNullColIsNullValue(lcv, rowIndex); - } + } + break; + case DOUBLE: { + DoubleColumnVector dcv = (DoubleColumnVector) batch.cols[offset + colIndex]; + if (writableCol != null) { + dcv.vector[rowIndex] = ((DoubleWritable) writableCol).get(); + dcv.isNull[rowIndex] = false; + } else { + dcv.vector[rowIndex] = Double.NaN; + setNullColIsNullValue(dcv, rowIndex); } - break; - case BINARY: { - BytesColumnVector bcv = (BytesColumnVector) batch.cols[off + i]; - if (writableCol != null) { - bcv.isNull[rowIndex] = false; - BytesWritable bw = (BytesWritable) writableCol; - byte[] bytes = bw.getBytes(); - int start = buffer.getLength(); - int length = bytes.length; - try { - buffer.write(bytes, 0, length); - } catch (IOException ioe) { - throw new IllegalStateException("bad write", ioe); - } - bcv.setRef(rowIndex, buffer.getData(), start, length); - } else { - setNullColIsNullValue(bcv, rowIndex); - } + } + break; + case TIMESTAMP: { + LongColumnVector lcv = (LongColumnVector) batch.cols[offset + colIndex]; + if (writableCol != null) { + Timestamp t = ((TimestampWritable) writableCol).getTimestamp(); + lcv.vector[rowIndex] = TimestampUtils.getTimeNanoSec(t); + lcv.isNull[rowIndex] = false; + } else { + lcv.vector[rowIndex] = 1; + setNullColIsNullValue(lcv, rowIndex); } - break; - case STRING: { - BytesColumnVector bcv = (BytesColumnVector) batch.cols[off + i]; - if (writableCol != null) { + } + break; + case BINARY: { + BytesColumnVector bcv = (BytesColumnVector) batch.cols[offset + colIndex]; + if (writableCol != null) { bcv.isNull[rowIndex] = false; - Text colText = (Text) writableCol; + BytesWritable bw = (BytesWritable) writableCol; + byte[] bytes = bw.getBytes(); int start = buffer.getLength(); - int length = colText.getLength(); + int length = bw.getLength(); try { - buffer.write(colText.getBytes(), 0, length); + buffer.write(bytes, 0, length); } catch (IOException ioe) { throw new IllegalStateException("bad write", ioe); } bcv.setRef(rowIndex, buffer.getData(), start, length); - } else { - setNullColIsNullValue(bcv, rowIndex); + } else { + setNullColIsNullValue(bcv, rowIndex); + } + } + break; + case STRING: { + BytesColumnVector bcv = (BytesColumnVector) batch.cols[offset + colIndex]; + if (writableCol != null) { + bcv.isNull[rowIndex] = false; + Text colText = (Text) writableCol; + int start = buffer.getLength(); + int length = colText.getLength(); + try { + buffer.write(colText.getBytes(), 0, length); + } catch (IOException ioe) { + throw new IllegalStateException("bad write", ioe); } + bcv.setRef(rowIndex, buffer.getData(), start, length); + } else { + setNullColIsNullValue(bcv, rowIndex); } - break; - case CHAR: { - BytesColumnVector bcv = (BytesColumnVector) batch.cols[off + i]; + } + break; + case CHAR: { + BytesColumnVector bcv = (BytesColumnVector) batch.cols[offset + colIndex]; + if (writableCol != null) { + bcv.isNull[rowIndex] = false; + HiveChar colHiveChar = ((HiveCharWritable) writableCol).getHiveChar(); + byte[] bytes = colHiveChar.getStrippedValue().getBytes(); + + // We assume the CHAR maximum length was enforced when the object was created. + int length = bytes.length; + + int start = buffer.getLength(); + try { + // In vector mode, we store CHAR as unpadded. + buffer.write(bytes, 0, length); + } catch (IOException ioe) { + throw new IllegalStateException("bad write", ioe); + } + bcv.setRef(rowIndex, buffer.getData(), start, length); + } else { + setNullColIsNullValue(bcv, rowIndex); + } + } + break; + case VARCHAR: { + BytesColumnVector bcv = (BytesColumnVector) batch.cols[offset + colIndex]; if (writableCol != null) { bcv.isNull[rowIndex] = false; - HiveChar colHiveChar = ((HiveCharWritable) writableCol).getHiveChar(); - byte[] bytes = colHiveChar.getStrippedValue().getBytes(); - - // We assume the CHAR maximum length was enforced when the object was created. + HiveVarchar colHiveVarchar = ((HiveVarcharWritable) writableCol).getHiveVarchar(); + byte[] bytes = colHiveVarchar.getValue().getBytes(); + + // We assume the VARCHAR maximum length was enforced when the object was created. int length = bytes.length; int start = buffer.getLength(); try { - // In vector mode, we store CHAR as unpadded. buffer.write(bytes, 0, length); } catch (IOException ioe) { throw new IllegalStateException("bad write", ioe); @@ -405,45 +476,21 @@ public static void addRowToBatchFrom(Object row, StructObjectInspector oi, } } break; - case VARCHAR: { - BytesColumnVector bcv = (BytesColumnVector) batch.cols[off + i]; - if (writableCol != null) { - bcv.isNull[rowIndex] = false; - HiveVarchar colHiveVarchar = ((HiveVarcharWritable) writableCol).getHiveVarchar(); - byte[] bytes = colHiveVarchar.getValue().getBytes(); - - // We assume the VARCHAR maximum length was enforced when the object was created. - int length = bytes.length; - - int start = buffer.getLength(); - try { - buffer.write(bytes, 0, length); - } catch (IOException ioe) { - throw new IllegalStateException("bad write", ioe); - } - bcv.setRef(rowIndex, buffer.getData(), start, length); - } else { - setNullColIsNullValue(bcv, rowIndex); - } - } - break; - case DECIMAL: - DecimalColumnVector dcv = (DecimalColumnVector) batch.cols[off + i]; - if (writableCol != null) { - dcv.isNull[rowIndex] = false; - HiveDecimalWritable wobj = (HiveDecimalWritable) writableCol; - dcv.vector[rowIndex].update(wobj.getHiveDecimal().unscaledValue(), - (short) wobj.getScale()); - } else { - setNullColIsNullValue(dcv, rowIndex); - } - break; - default: - throw new HiveException("Vectorizaton is not supported for datatype:" - + poi.getPrimitiveCategory()); + case DECIMAL: + DecimalColumnVector dcv = (DecimalColumnVector) batch.cols[offset + colIndex]; + if (writableCol != null) { + dcv.isNull[rowIndex] = false; + HiveDecimalWritable wobj = (HiveDecimalWritable) writableCol; + dcv.vector[rowIndex].update(wobj.getHiveDecimal().unscaledValue(), + (short) wobj.getScale()); + } else { + setNullColIsNullValue(dcv, rowIndex); } + break; + default: + throw new HiveException("Vectorizaton is not supported for datatype:" + + poi.getPrimitiveCategory()); } } - } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java index 21c757e..4f57aac 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java @@ -22,10 +22,12 @@ import java.sql.Timestamp; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -45,6 +47,7 @@ import org.apache.hadoop.hive.serde2.Deserializer; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.SerDeUtils; +import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; @@ -83,7 +86,11 @@ private Map partitionValues; //partition types - private Map partitionTypes; + private Map partitionTypes; + + // partition column positions, for use by classes that need to know whether a given column is a + // partition column + private Set partitionCols; // Column projection list - List of column indexes to include. This // list does not contain partition columns @@ -132,7 +139,7 @@ public VectorizedRowBatchCtx() { public void init(Configuration hiveConf, String fileKey, StructObjectInspector rowOI) { Map> scratchColumnVectorTypes = - Utilities.getScratchColumnVectorTypes(hiveConf); + Utilities.getAllScratchColumnVectorTypeMaps(hiveConf); columnTypeMap = scratchColumnVectorTypes.get(fileKey); this.rowOI= rowOI; this.rawRowOI = rowOI; @@ -183,7 +190,7 @@ public void init(Configuration hiveConf, FileSplit split) throws ClassNotFoundEx String partitionPath = split.getPath().getParent().toString(); columnTypeMap = Utilities - .getScratchColumnVectorTypes(hiveConf) + .getAllScratchColumnVectorTypeMaps(hiveConf) .get(partitionPath); Properties partProps = @@ -202,12 +209,13 @@ public void init(Configuration hiveConf, FileSplit split) throws ClassNotFoundEx // Check to see if this split is part of a partition of a table String pcols = partProps.getProperty(hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS); + String[] partKeys = null; if (pcols != null && pcols.length() > 0) { // Partitions exist for this table. Get the partition object inspector and // raw row object inspector (row with out partition col) LinkedHashMap partSpec = part.getPartSpec(); - String[] partKeys = pcols.trim().split("/"); + partKeys = pcols.trim().split("/"); String pcolTypes = partProps.getProperty(hive_metastoreConstants.META_TABLE_PARTITION_COLUMN_TYPES); String[] partKeyTypes = pcolTypes.trim().split(":"); @@ -261,6 +269,15 @@ public void init(Configuration hiveConf, FileSplit split) throws ClassNotFoundEx .asList(new StructObjectInspector[] {partRawRowObjectInspector, partObjectInspector})); rowOI = rowObjectInspector; rawRowOI = partRawRowObjectInspector; + + // We have to do this after we've set rowOI, as getColIndexBasedOnColName uses it + partitionCols = new HashSet(); + if (pcols != null && pcols.length() > 0) { + for (int i = 0; i < partKeys.length; i++) { + partitionCols.add(getColIndexBasedOnColName(partKeys[i])); + } + } + } else { // No partitions for this table, hence row OI equals raw row OI @@ -487,7 +504,7 @@ public void addPartitionColsToBatch(VectorizedRowBatch batch) throws HiveExcepti lcv.isNull[0] = true; lcv.isRepeating = true; } else { - lcv.fill(((Date) value).getTime()); + lcv.fill(DateWritable.dateToDays((Date) value)); lcv.isNull[0] = false; } } @@ -500,7 +517,7 @@ public void addPartitionColsToBatch(VectorizedRowBatch batch) throws HiveExcepti lcv.isNull[0] = true; lcv.isRepeating = true; } else { - lcv.fill((long)(((Timestamp) value).getTime())); + lcv.fill(TimestampUtils.getTimeNanoSec((Timestamp) value)); lcv.isNull[0] = false; } } @@ -585,6 +602,16 @@ public void addPartitionColsToBatch(VectorizedRowBatch batch) throws HiveExcepti } } + /** + * Determine whether a given column is a partition column + * @param colnum column number in + * {@link org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch}s created by this context. + * @return true if it is a partition column, false otherwise + */ + public final boolean isPartitionCol(int colnum) { + return (partitionCols == null) ? false : partitionCols.contains(colnum); + } + private void addScratchColumnsToBatch(VectorizedRowBatch vrb) throws HiveException { if (columnTypeMap != null && !columnTypeMap.isEmpty()) { int origNumCols = vrb.numCols; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/index/IndexPredicateAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/index/IndexPredicateAnalyzer.java index 683618f..a29601c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/index/IndexPredicateAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/index/IndexPredicateAnalyzer.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.index; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; @@ -44,6 +45,13 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToBinary; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToChar; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToDate; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToDecimal; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToUnixTimeStamp; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToUtcTimestamp; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToVarchar; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseCompare; @@ -57,14 +65,14 @@ public class IndexPredicateAnalyzer { private final Set udfNames; - private final Set allowedColumnNames; + private final Map> columnToUDFs; private FieldValidator fieldValidator; private boolean acceptsFields; public IndexPredicateAnalyzer() { udfNames = new HashSet(); - allowedColumnNames = new HashSet(); + columnToUDFs = new HashMap>(); } public void setFieldValidator(FieldValidator fieldValidator) { @@ -89,7 +97,7 @@ public void addComparisonOp(String udfName) { * column names are allowed.) */ public void clearAllowedColumnNames() { - allowedColumnNames.clear(); + columnToUDFs.clear(); } /** @@ -98,7 +106,22 @@ public void clearAllowedColumnNames() { * @param columnName name of column to be allowed */ public void allowColumnName(String columnName) { - allowedColumnNames.add(columnName); + columnToUDFs.put(columnName, udfNames); + } + + /** + * add allowed functions per column + * @param columnName + * @param udfs + */ + public void addComparisonOp(String columnName, String... udfs) { + Set allowed = columnToUDFs.get(columnName); + if (allowed == null || allowed == udfNames) { + // override + columnToUDFs.put(columnName, new HashSet(Arrays.asList(udfs))); + } else { + allowed.addAll(Arrays.asList(udfs)); + } } /** @@ -152,6 +175,32 @@ public Object process(Node nd, Stack stack, return residualPredicate; } + //Check if ExprNodeColumnDesc is wrapped in expr. + //If so, peel off. Otherwise return itself. + private ExprNodeDesc getColumnExpr(ExprNodeDesc expr) { + if (expr instanceof ExprNodeColumnDesc) { + return expr; + } + ExprNodeGenericFuncDesc funcDesc = null; + if (expr instanceof ExprNodeGenericFuncDesc) { + funcDesc = (ExprNodeGenericFuncDesc) expr; + } + if (null == funcDesc) { + return expr; + } + GenericUDF udf = funcDesc.getGenericUDF(); + // check if its a simple cast expression. + if ((udf instanceof GenericUDFBridge || udf instanceof GenericUDFToBinary + || udf instanceof GenericUDFToChar || udf instanceof GenericUDFToVarchar + || udf instanceof GenericUDFToDecimal || udf instanceof GenericUDFToDate + || udf instanceof GenericUDFToUnixTimeStamp || udf instanceof GenericUDFToUtcTimestamp) + && funcDesc.getChildren().size() == 1 + && funcDesc.getChildren().get(0) instanceof ExprNodeColumnDesc) { + return expr.getChildren().get(0); + } + return expr; + } + private ExprNodeDesc analyzeExpr( ExprNodeGenericFuncDesc expr, List searchConditions, @@ -182,11 +231,17 @@ private ExprNodeDesc analyzeExpr( } ExprNodeDesc expr1 = (ExprNodeDesc) nodeOutputs[0]; ExprNodeDesc expr2 = (ExprNodeDesc) nodeOutputs[1]; + // We may need to peel off the GenericUDFBridge that is added by CBO or user + if (expr1.getTypeInfo().equals(expr2.getTypeInfo())) { + expr1 = getColumnExpr(expr1); + expr2 = getColumnExpr(expr2); + } + ExprNodeDesc[] extracted = ExprNodeDescUtils.extractComparePair(expr1, expr2); if (extracted == null || (extracted.length > 2 && !acceptsFields)) { return expr; } - + ExprNodeColumnDesc columnDesc; ExprNodeConstantDesc constantDesc; if (extracted[0] instanceof ExprNodeConstantDesc) { @@ -198,12 +253,13 @@ private ExprNodeDesc analyzeExpr( constantDesc = (ExprNodeConstantDesc) extracted[1]; } - String udfName = genericUDF.getUdfName(); - if (!udfNames.contains(genericUDF.getUdfName())) { + Set allowed = columnToUDFs.get(columnDesc.getColumn()); + if (allowed == null) { return expr; } - if (!allowedColumnNames.contains(columnDesc.getColumn())) { + String udfName = genericUDF.getUdfName(); + if (!allowed.contains(genericUDF.getUdfName())) { return expr; } @@ -216,6 +272,13 @@ private ExprNodeDesc analyzeExpr( fields = ExprNodeDescUtils.extractFields(fieldDesc); } + // We also need to update the expr so that the index query can be generated. + // Note that, hive does not support UDFToDouble etc in the query text. + List list = new ArrayList(); + list.add(expr1); + list.add(expr2); + expr = new ExprNodeGenericFuncDesc(expr.getTypeInfo(), expr.getGenericUDF(), list); + searchConditions.add( new IndexSearchCondition( columnDesc, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidInputFormat.java index 2f63524..e1d2395 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidInputFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidInputFormat.java @@ -155,6 +155,8 @@ public Reporter getReporter() { public static interface RawReader extends RecordReader { public ObjectInspector getObjectInspector(); + + public boolean isDelete(V value); } /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java index 928fd61..bede378 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java @@ -348,7 +348,7 @@ public static Directory getAcidState(Path directory, long bestBaseTxn = 0; final List deltas = new ArrayList(); List working = new ArrayList(); - final List original = new ArrayList(); + List originalDirectories = new ArrayList(); final List obsolete = new ArrayList(); List children = SHIMS.listLocatedStatus(fs, directory, hiddenFileFilter); @@ -375,16 +375,26 @@ public static Directory getAcidState(Path directory, working.add(delta); } } else { - findOriginals(fs, child, original); + // This is just the directory. We need to recurse and find the actual files. But don't + // do this until we have determined there is no base. This saves time. Plus, + // it is possible that the cleaner is running and removing these original files, + // in which case recursing through them could cause us to get an error. + originalDirectories.add(child); } } + final List original = new ArrayList(); // if we have a base, the original files are obsolete. if (bestBase != null) { - obsolete.addAll(original); // remove the entries so we don't get confused later and think we should // use them. original.clear(); + } else { + // Okay, we're going to need these originals. Recurse through them and figure out what we + // really need. + for (FileStatus origDir : originalDirectories) { + findOriginals(fs, origDir, original); + } } Collections.sort(working); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java index 13c9751..a6a8176 100755 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java @@ -275,7 +275,7 @@ private void addSplitsForGroup(List dirs, TableScanOperator tableScan, Job InputFormat inputFormat, Class inputFormatClass, int splits, TableDesc table, List result) throws IOException { - Utilities.copyTableJobPropertiesToConf(table, conf); + Utilities.copyTablePropertiesToConf(table, conf); if (tableScan != null) { pushFilters(conf, tableScan); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java b/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java index 7b6df27..5ba6612 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java @@ -21,8 +21,6 @@ import java.util.HashMap; import java.util.Map; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; @@ -47,8 +45,6 @@ }; private static Map inputNameIOContextMap = new HashMap(); - private static IOContext ioContext = new IOContext(); - public static Map getMap() { return inputNameIOContextMap; } @@ -72,6 +68,7 @@ public static IOContext get(Configuration conf) { public static void clear() { IOContext.threadLocal.remove(); + inputNameIOContextMap.clear(); } long currentBlockStart; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java index 9007771..9340ce6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java @@ -388,6 +388,7 @@ public boolean validateInput(FileSystem fs, HiveConf conf, ConfVars.HIVE_ORC_INCLUDE_FILE_FOOTER_IN_SPLITS); numBuckets = Math.max(conf.getInt(hive_metastoreConstants.BUCKET_COUNT, 0), 0); + LOG.debug("Number of buckets specified by conf file is " + numBuckets); int cacheStripeDetailsSize = HiveConf.getIntVar(conf, ConfVars.HIVE_ORC_CACHE_STRIPE_DETAILS_SIZE); int numThreads = HiveConf.getIntVar(conf, @@ -1134,7 +1135,7 @@ public float getProgress() throws IOException { @Override public ObjectInspector getObjectInspector() { - return ((StructObjectInspector) reader.getObjectInspector()) + return ((StructObjectInspector) records.getObjectInspector()) .getAllStructFieldRefs().get(OrcRecordUpdater.ROW) .getFieldObjectInspector(); } @@ -1188,7 +1189,9 @@ static Path findOriginalBucket(FileSystem fs, int bucket) throws IOException { for(FileStatus stat: fs.listStatus(directory)) { String name = stat.getPath().getName(); - if (Integer.parseInt(name.substring(0, name.indexOf('_'))) == bucket) { + String numberPart = name.substring(0, name.indexOf('_')); + if (org.apache.commons.lang3.StringUtils.isNumeric(numberPart) && + Integer.parseInt(numberPart) == bucket) { return stat.getPath(); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java index 74ea23b..144f21e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hive.ql.io.AcidInputFormat; import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.io.RecordIdentifier; +import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; @@ -37,9 +38,10 @@ import org.apache.hadoop.io.LongWritable; import java.io.IOException; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; +import java.util.Deque; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -448,6 +450,10 @@ private void discoverKeyBounds(Reader reader, // we always want to read all of the deltas eventOptions.range(0, Long.MAX_VALUE); + // Turn off the sarg before pushing it to delta. We never want to push a sarg to a delta as + // it can produce wrong results (if the latest valid version of the record is filtered out by + // the sarg) or ArrayOutOfBounds errors (when the sarg is applied to a delete record) + eventOptions.searchArgument(null, null); if (deltaDirectory != null) { for(Path delta: deltaDirectory) { ReaderKey key = new ReaderKey(); @@ -627,8 +633,16 @@ public ObjectInspector getObjectInspector() { // Parse the configuration parameters ArrayList columnNames = new ArrayList(); + Deque virtualColumns = new ArrayDeque(); if (columnNameProperty != null && columnNameProperty.length() > 0) { - Collections.addAll(columnNames, columnNameProperty.split(",")); + String[] colNames = columnNameProperty.split(","); + for (int i = 0; i < colNames.length; i++) { + if (VirtualColumn.VIRTUAL_COLUMN_NAMES.contains(colNames[i])) { + virtualColumns.addLast(i); + } else { + columnNames.add(colNames[i]); + } + } } if (columnTypeProperty == null) { // Default type: all string @@ -644,6 +658,9 @@ public ObjectInspector getObjectInspector() { ArrayList fieldTypes = TypeInfoUtils.getTypeInfosFromTypeString(columnTypeProperty); + while (virtualColumns.size() > 0) { + fieldTypes.remove(virtualColumns.removeLast()); + } StructTypeInfo rowType = new StructTypeInfo(); rowType.setAllStructFieldNames(columnNames); rowType.setAllStructFieldTypeInfos(fieldTypes); @@ -651,6 +668,11 @@ public ObjectInspector getObjectInspector() { (OrcStruct.createObjectInspector(rowType)); } + @Override + public boolean isDelete(OrcStruct value) { + return OrcRecordUpdater.getOperation(value) == OrcRecordUpdater.DELETE_OPERATION; + } + /** * Get the number of columns in the underlying rows. * @return 0 if there are no base and no deltas. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java index c4268c1..a381443 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java @@ -27,8 +27,10 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.VectorizedSerde; +import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.SerDe; import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.SerDeSpec; import org.apache.hadoop.hive.serde2.SerDeStats; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; @@ -40,6 +42,7 @@ * A serde class for ORC. * It transparently passes the object to/from the ORC file reader/writer. */ +@SerDeSpec(schemaProps = {serdeConstants.LIST_COLUMNS, serdeConstants.LIST_COLUMN_TYPES}) public class OrcSerde implements SerDe, VectorizedSerde { private static final Log LOG = LogFactory.getLog(OrcSerde.class); @@ -75,9 +78,9 @@ Object getRow() { @Override public void initialize(Configuration conf, Properties table) { // Read the configuration parameters - String columnNameProperty = table.getProperty("columns"); + String columnNameProperty = table.getProperty(serdeConstants.LIST_COLUMNS); // NOTE: if "columns.types" is missing, all columns will be of String type - String columnTypeProperty = table.getProperty("columns.types"); + String columnTypeProperty = table.getProperty(serdeConstants.LIST_COLUMN_TYPES); // Parse the configuration parameters ArrayList columnNames = new ArrayList(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java index 67c19ad..bdc2806 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java @@ -47,6 +47,7 @@ import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector; import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.TimestampUtils; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr; import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf; @@ -1074,41 +1075,20 @@ Object nextVector(Object previousVector, long batchSize) throws IOException { result = (LongColumnVector) previousVector; } - // Read present/isNull stream - super.nextVector(result, batchSize); - - data.nextVector(result, batchSize); - nanoVector.isNull = result.isNull; - nanos.nextVector(nanoVector, batchSize); - - if(result.isRepeating && nanoVector.isRepeating) { - batchSize = 1; - } - - // Non repeating values preset in the vector. Iterate thru the vector and populate the time + result.reset(); + Object obj = null; for (int i = 0; i < batchSize; i++) { - if (!result.isNull[i]) { - long ms = (result.vector[result.isRepeating ? 0 : i] + WriterImpl.BASE_TIMESTAMP) - * WriterImpl.MILLIS_PER_SECOND; - long ns = parseNanos(nanoVector.vector[nanoVector.isRepeating ? 0 : i]); - // the rounding error exists because java always rounds up when dividing integers - // -42001/1000 = -42; and -42001 % 1000 = -1 (+ 1000) - // to get the correct value we need - // (-42 - 1)*1000 + 999 = -42001 - // (42)*1000 + 1 = 42001 - if(ms < 0 && ns != 0) { - ms -= 1000; - } - // Convert millis into nanos and add the nano vector value to it - result.vector[i] = (ms * 1000000) + ns; + obj = next(obj); + if (obj == null) { + result.noNulls = false; + result.isNull[i] = true; + } else { + TimestampWritable writable = (TimestampWritable) obj; + Timestamp timestamp = writable.getTimestamp(); + result.vector[i] = TimestampUtils.getTimeNanoSec(timestamp); } } - if(!(result.isRepeating && nanoVector.isRepeating)) { - // both have to repeat for the result to be repeating - result.isRepeating = false; - } - return result; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowReader.java index ca90fc5..a8e5c2e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowReader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowReader.java @@ -48,7 +48,6 @@ private final OrcStruct value; private final VectorizedRowBatchCtx rowBatchCtx; private final ObjectInspector objectInspector; - private boolean needToSetPartition = true; private final DataOutputBuffer buffer = new DataOutputBuffer(); VectorizedOrcAcidRowReader(AcidInputFormat.RowReader inner, @@ -83,23 +82,20 @@ public boolean next(NullWritable nullWritable, if (!innerReader.next(key, value)) { return false; } - if (needToSetPartition) { - try { - rowBatchCtx.addPartitionColsToBatch(vectorizedRowBatch); - } catch (HiveException e) { - throw new IOException("Problem adding partition column", e); - } - needToSetPartition = false; + try { + rowBatchCtx.addPartitionColsToBatch(vectorizedRowBatch); + } catch (HiveException e) { + throw new IOException("Problem adding partition column", e); } try { - VectorizedBatchUtil.addRowToBatch(value, + VectorizedBatchUtil.acidAddRowToBatch(value, (StructObjectInspector) objectInspector, - vectorizedRowBatch.size++, vectorizedRowBatch, buffer); + vectorizedRowBatch.size++, vectorizedRowBatch, rowBatchCtx, buffer); while (vectorizedRowBatch.size < vectorizedRowBatch.selected.length && innerReader.next(key, value)) { - VectorizedBatchUtil.addRowToBatch(value, + VectorizedBatchUtil.acidAddRowToBatch(value, (StructObjectInspector) objectInspector, - vectorizedRowBatch.size++, vectorizedRowBatch, buffer); + vectorizedRowBatch.size++, vectorizedRowBatch, rowBatchCtx, buffer); } } catch (HiveException he) { throw new IOException("error iterating", he); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetOutputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetOutputFormat.java index 30c91ea..ea4109d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetOutputFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetOutputFormat.java @@ -21,6 +21,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.io.HiveOutputFormat; @@ -40,6 +41,8 @@ import org.apache.hadoop.util.Progressable; import parquet.hadoop.ParquetOutputFormat; +import parquet.hadoop.metadata.CompressionCodecName; +import parquet.hadoop.util.ContextUtil; /** * @@ -110,15 +113,19 @@ public void checkOutputSpecs(final FileSystem ignored, final JobConf job) throws } DataWritableWriteSupport.setSchema(HiveSchemaConverter.convert(columnNames, columnTypes), jobConf); - return getParquerRecordWriterWrapper(realOutputFormat, jobConf, finalOutPath.toString(), progress); + + return getParquerRecordWriterWrapper(realOutputFormat, jobConf, finalOutPath.toString(), + progress,tableProperties); } protected ParquetRecordWriterWrapper getParquerRecordWriterWrapper( ParquetOutputFormat realOutputFormat, JobConf jobConf, String finalOutPath, - Progressable progress + Progressable progress, + Properties tableProperties ) throws IOException { - return new ParquetRecordWriterWrapper(realOutputFormat, jobConf, finalOutPath.toString(), progress); + return new ParquetRecordWriterWrapper(realOutputFormat, jobConf, finalOutPath.toString(), + progress,tableProperties); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java index 2ad7330..3f8e4d7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java @@ -75,6 +75,7 @@ final Map keyValueMetaData, final MessageType fileSchema) { final String columns = configuration.get(IOConstants.COLUMNS); final Map contextMetadata = new HashMap(); + final boolean indexAccess = configuration.getBoolean(PARQUET_COLUMN_INDEX_ACCESS, false); if (columns != null) { final List listColumns = getColumns(columns); final Map lowerCaseFileSchemaColumns = new HashMap(); @@ -82,45 +83,50 @@ lowerCaseFileSchemaColumns.put(c.getPath()[0].toLowerCase(), c.getPath()[0]); } final List typeListTable = new ArrayList(); - for (String col : listColumns) { - col = col.toLowerCase(); - // listColumns contains partition columns which are metadata only - if (lowerCaseFileSchemaColumns.containsKey(col)) { - typeListTable.add(fileSchema.getType(lowerCaseFileSchemaColumns.get(col))); - } else { - // below allows schema evolution - typeListTable.add(new PrimitiveType(Repetition.OPTIONAL, PrimitiveTypeName.BINARY, col)); + if(indexAccess) { + for (int index = 0; index < listColumns.size(); index++) { + //Take columns based on index or pad the field + if(index < fileSchema.getFieldCount()) { + typeListTable.add(fileSchema.getType(index)); + } else { + //prefixing with '_mask_' to ensure no conflict with named + //columns in the file schema + typeListTable.add(new PrimitiveType(Repetition.OPTIONAL, PrimitiveTypeName.BINARY, "_mask_"+listColumns.get(index))); + } + } + } else { + for (String col : listColumns) { + col = col.toLowerCase(); + // listColumns contains partition columns which are metadata only + if (lowerCaseFileSchemaColumns.containsKey(col)) { + typeListTable.add(fileSchema.getType(lowerCaseFileSchemaColumns.get(col))); + } else { + // below allows schema evolution + typeListTable.add(new PrimitiveType(Repetition.OPTIONAL, PrimitiveTypeName.BINARY, col)); + } } } MessageType tableSchema = new MessageType(TABLE_SCHEMA, typeListTable); contextMetadata.put(HIVE_SCHEMA_KEY, tableSchema.toString()); - MessageType requestedSchemaByUser = tableSchema; final List indexColumnsWanted = ColumnProjectionUtils.getReadColumnIDs(configuration); final List typeListWanted = new ArrayList(); - final boolean indexAccess = configuration.getBoolean(PARQUET_COLUMN_INDEX_ACCESS, false); + for (final Integer idx : indexColumnsWanted) { if (idx < listColumns.size()) { String col = listColumns.get(idx); if (indexAccess) { - typeListWanted.add(tableSchema.getType(col)); + typeListWanted.add(fileSchema.getFields().get(idx)); } else { col = col.toLowerCase(); if (lowerCaseFileSchemaColumns.containsKey(col)) { typeListWanted.add(tableSchema.getType(lowerCaseFileSchemaColumns.get(col))); - } else { - // should never occur? - String msg = "Column " + col + " at index " + idx + " does not exist in " + - lowerCaseFileSchemaColumns; - throw new IllegalStateException(msg); } } } } - requestedSchemaByUser = resolveSchemaAccess(new MessageType(fileSchema.getName(), - typeListWanted), fileSchema, configuration); - + MessageType requestedSchemaByUser = new MessageType(fileSchema.getName(), typeListWanted); return new ReadContext(requestedSchemaByUser, contextMetadata); } else { contextMetadata.put(HIVE_SCHEMA_KEY, fileSchema.toString()); @@ -147,26 +153,7 @@ throw new IllegalStateException("ReadContext not initialized properly. " + "Don't know the Hive Schema."); } - final MessageType tableSchema = resolveSchemaAccess(MessageTypeParser. - parseMessageType(metadata.get(HIVE_SCHEMA_KEY)), fileSchema, configuration); + final MessageType tableSchema = MessageTypeParser.parseMessageType(metadata.get(HIVE_SCHEMA_KEY)); return new DataWritableRecordConverter(readContext.getRequestedSchema(), tableSchema); } - - /** - * Determine the file column names based on the position within the requested columns and - * use that as the requested schema. - */ - private MessageType resolveSchemaAccess(MessageType requestedSchema, MessageType fileSchema, - Configuration configuration) { - if (configuration.getBoolean(PARQUET_COLUMN_INDEX_ACCESS, false)) { - final List listColumns = getColumns(configuration.get(IOConstants.COLUMNS)); - List requestedTypes = new ArrayList(); - for(Type t : requestedSchema.getFields()) { - int index = listColumns.indexOf(t.getName()); - requestedTypes.add(fileSchema.getType(index)); - } - requestedSchema = new MessageType(requestedSchema.getName(), requestedTypes); - } - return requestedSchema; - } -} \ No newline at end of file +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java index e5c663e..d63fc72 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java @@ -23,9 +23,10 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.type.HiveDecimal; -import org.apache.hadoop.hive.ql.io.IOConstants; +import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.AbstractSerDe; import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.SerDeSpec; import org.apache.hadoop.hive.serde2.SerDeStats; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; @@ -69,6 +70,7 @@ * A ParquetHiveSerDe for Hive (with the deprecated package mapred) * */ +@SerDeSpec(schemaProps = {serdeConstants.LIST_COLUMNS, serdeConstants.LIST_COLUMN_TYPES}) public class ParquetHiveSerDe extends AbstractSerDe { public static final Text MAP_KEY = new Text("key"); public static final Text MAP_VALUE = new Text("value"); @@ -105,8 +107,8 @@ public final void initialize(final Configuration conf, final Properties tbl) thr final List columnNames; final List columnTypes; // Get column names and sort order - final String columnNameProperty = tbl.getProperty(IOConstants.COLUMNS); - final String columnTypeProperty = tbl.getProperty(IOConstants.COLUMNS_TYPES); + final String columnNameProperty = tbl.getProperty(serdeConstants.LIST_COLUMNS); + final String columnTypeProperty = tbl.getProperty(serdeConstants.LIST_COLUMN_TYPES); if (columnNameProperty.length() == 0) { columnNames = new ArrayList(); @@ -242,7 +244,7 @@ private Writable createPrimitive(final Object obj, final PrimitiveObjectInspecto case BOOLEAN: return new BooleanWritable(((BooleanObjectInspector) inspector).get(obj) ? Boolean.TRUE : Boolean.FALSE); case BYTE: - return new ByteWritable((byte) ((ByteObjectInspector) inspector).get(obj)); + return new ByteWritable(((ByteObjectInspector) inspector).get(obj)); case DOUBLE: return new DoubleWritable(((DoubleObjectInspector) inspector).get(obj)); case FLOAT: @@ -252,7 +254,7 @@ private Writable createPrimitive(final Object obj, final PrimitiveObjectInspecto case LONG: return new LongWritable(((LongObjectInspector) inspector).get(obj)); case SHORT: - return new ShortWritable((short) ((ShortObjectInspector) inspector).get(obj)); + return new ShortWritable(((ShortObjectInspector) inspector).get(obj)); case STRING: String v = ((StringObjectInspector) inspector).getPrimitiveJavaObject(obj); try { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/NanoTime.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/NanoTime.java index 85cf5e7..fb2a6c2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/NanoTime.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/NanoTime.java @@ -14,6 +14,7 @@ package org.apache.hadoop.hive.ql.io.parquet.timestamp; import java.nio.ByteBuffer; +import java.nio.ByteOrder; import parquet.Preconditions; import parquet.io.api.Binary; @@ -28,7 +29,10 @@ public static NanoTime fromBinary(Binary bytes) { Preconditions.checkArgument(bytes.length() == 12, "Must be 12 bytes"); ByteBuffer buf = bytes.toByteBuffer(); - return new NanoTime(buf.getInt(), buf.getLong()); + buf.order(ByteOrder.LITTLE_ENDIAN); + long timeOfDayNanos = buf.getLong(); + int julianDay = buf.getInt(); + return new NanoTime(julianDay, timeOfDayNanos); } public NanoTime(int julianDay, long timeOfDayNanos) { @@ -46,8 +50,9 @@ public long getTimeOfDayNanos() { public Binary toBinary() { ByteBuffer buf = ByteBuffer.allocate(12); - buf.putInt(julianDay); + buf.order(ByteOrder.LITTLE_ENDIAN); buf.putLong(timeOfDayNanos); + buf.putInt(julianDay); buf.flip(); return Binary.fromByteBuffer(buf); } @@ -60,4 +65,4 @@ public void writeValue(RecordConsumer recordConsumer) { public String toString() { return "NanoTime{julianDay="+julianDay+", timeOfDayNanos="+timeOfDayNanos+"}"; } -} \ No newline at end of file +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/write/ParquetRecordWriterWrapper.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/write/ParquetRecordWriterWrapper.java index 07b003a..765b5ac 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/write/ParquetRecordWriterWrapper.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/write/ParquetRecordWriterWrapper.java @@ -14,6 +14,7 @@ package org.apache.hadoop.hive.ql.io.parquet.write; import java.io.IOException; +import java.util.Properties; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -29,6 +30,7 @@ import org.apache.hadoop.util.Progressable; import parquet.hadoop.ParquetOutputFormat; +import parquet.hadoop.metadata.CompressionCodecName; import parquet.hadoop.util.ContextUtil; public class ParquetRecordWriterWrapper implements RecordWriter, @@ -43,7 +45,8 @@ public ParquetRecordWriterWrapper( final OutputFormat realOutputFormat, final JobConf jobConf, final String name, - final Progressable progress) throws IOException { + final Progressable progress, Properties tableProperties) throws + IOException { try { // create a TaskInputOutputContext TaskAttemptID taskAttemptID = TaskAttemptID.forName(jobConf.get("mapred.task.id")); @@ -53,7 +56,21 @@ public ParquetRecordWriterWrapper( taskContext = ContextUtil.newTaskAttemptContext(jobConf, taskAttemptID); LOG.info("creating real writer to write at " + name); - realWriter = ((ParquetOutputFormat) realOutputFormat).getRecordWriter(taskContext, new Path(name)); + + String compressionName = tableProperties.getProperty(ParquetOutputFormat.COMPRESSION); + if (compressionName != null && !compressionName.isEmpty()) { + //get override compression properties via "tblproperties" clause if it is set + LOG.debug("get override compression properties via tblproperties"); + + ContextUtil.getConfiguration(taskContext); + CompressionCodecName codecName = CompressionCodecName.fromConf(compressionName); + realWriter = ((ParquetOutputFormat) realOutputFormat).getRecordWriter(jobConf, + new Path(name), codecName); + } else { + realWriter = ((ParquetOutputFormat) realOutputFormat).getRecordWriter(taskContext, + new Path(name)); + } + LOG.info("real writer: " + realWriter); } catch (final InterruptedException e) { throw new IOException(e); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java index 6aa35e1..eeb9641 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java @@ -18,17 +18,21 @@ package org.apache.hadoop.hive.ql.io.sarg; -import com.esotericsoftware.kryo.Kryo; -import com.esotericsoftware.kryo.io.Input; -import com.esotericsoftware.kryo.io.Output; +import java.math.BigDecimal; +import java.sql.Timestamp; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.apache.commons.codec.binary.Base64; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.common.type.HiveChar; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.common.type.HiveVarchar; -import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf; -import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; @@ -52,15 +56,9 @@ import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import java.math.BigDecimal; -import java.sql.Timestamp; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Deque; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; /** * The implementation of SearchArguments. @@ -366,7 +364,7 @@ private static Object boxLiteral(ExprNodeConstantDesc lit) { case STRING: return StringUtils.stripEnd(lit.getValue().toString(), null); case FLOAT: - return ((Number) lit.getValue()).doubleValue(); + return Double.parseDouble(lit.getValue().toString()); case DATE: case TIMESTAMP: case DECIMAL: @@ -977,7 +975,9 @@ private static Object boxLiteral(Object literal) { literal instanceof Integer) { return Long.valueOf(literal.toString()); } else if (literal instanceof Float) { - return Double.valueOf((Float) literal); + // to avoid change in precision when upcasting float to double + // we convert the literal to string and parse it as double. (HIVE-8460) + return Double.parseDouble(literal.toString()); } else { throw new IllegalArgumentException("Unknown type for literal " + literal); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index cbbe781..b900627 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -313,7 +313,7 @@ public void dropDatabase(String name, boolean deleteData, boolean ignoreUnknownD * @param name * @param deleteData * @param ignoreUnknownDb if true, will ignore NoSuchObjectException - * @param cascade if true, delete all tables on the DB if exists. Othewise, the query + * @param cascade if true, delete all tables on the DB if exists. Otherwise, the query * will fail if table still exists. * @throws HiveException * @throws NoSuchObjectException @@ -331,7 +331,7 @@ public void dropDatabase(String name, boolean deleteData, boolean ignoreUnknownD /** - * Creates a table metdata and the directory for the table data + * Creates a table metadata and the directory for the table data * * @param tableName * name of the table @@ -355,7 +355,7 @@ public void createTable(String tableName, List columns, } /** - * Creates a table metdata and the directory for the table data + * Creates a table metadata and the directory for the table data * * @param tableName * name of the table @@ -885,16 +885,21 @@ public Index getIndex(String dbName, String baseTableName, } } - public boolean dropIndex(String baseTableName, String index_name, boolean deleteData) throws HiveException { + public boolean dropIndex(String baseTableName, String index_name, + boolean throwException, boolean deleteData) throws HiveException { String[] names = Utilities.getDbTableName(baseTableName); - return dropIndex(names[0], names[1], index_name, deleteData); + return dropIndex(names[0], names[1], index_name, throwException, deleteData); } - public boolean dropIndex(String db_name, String tbl_name, String index_name, boolean deleteData) throws HiveException { + public boolean dropIndex(String db_name, String tbl_name, String index_name, + boolean throwException, boolean deleteData) throws HiveException { try { return getMSC().dropIndex(db_name, tbl_name, index_name, deleteData); } catch (NoSuchObjectException e) { - throw new HiveException("Partition or table doesn't exist. " + e.getMessage(), e); + if (throwException) { + throw new HiveException("Index " + index_name + " doesn't exist. ", e); + } + return false; } catch (Exception e) { throw new HiveException(e.getMessage(), e); } @@ -2861,10 +2866,6 @@ public GetOpenTxnsInfoResponse showTransactions() throws HiveException { } } - public static String[] getQualifiedNames(String qualifiedName) { - return qualifiedName.split("\\."); - } - public void createFunction(Function func) throws HiveException { try { getMSC().createFunction(func); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index 2bbedd3..211ab6c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -34,7 +34,6 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.MetaStoreUtils; @@ -192,7 +191,7 @@ public void checkValidity() throws HiveException { "at least one column must be specified for the table"); } if (!isView()) { - if (null == getDeserializerFromMetaStore()) { + if (null == getDeserializerFromMetaStore(false)) { throw new HiveException("must specify a non-null serDe"); } if (null == getInputFormatClass()) { @@ -253,14 +252,25 @@ final public Path getDataLocation() { final public Deserializer getDeserializer() { if (deserializer == null) { - deserializer = getDeserializerFromMetaStore(); + deserializer = getDeserializerFromMetaStore(false); } return deserializer; } - private Deserializer getDeserializerFromMetaStore() { + final public Class getDeserializerClass() throws Exception { + return MetaStoreUtils.getDeserializerClass(Hive.get().getConf(), tTable); + } + + final public Deserializer getDeserializer(boolean skipConfError) { + if (deserializer == null) { + deserializer = getDeserializerFromMetaStore(skipConfError); + } + return deserializer; + } + + final public Deserializer getDeserializerFromMetaStore(boolean skipConfError) { try { - return MetaStoreUtils.getDeserializer(Hive.get().getConf(), tTable); + return MetaStoreUtils.getDeserializer(Hive.get().getConf(), tTable, skipConfError); } catch (MetaException e) { throw new RuntimeException(e); } catch (HiveException e) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java index 8e17475..046a52f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hive.ql.exec.GroupByOperator; import org.apache.hadoop.hive.ql.exec.LateralViewForwardOperator; import org.apache.hadoop.hive.ql.exec.LateralViewJoinOperator; +import org.apache.hadoop.hive.ql.exec.LimitOperator; import org.apache.hadoop.hive.ql.exec.MapJoinOperator; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.PTFOperator; @@ -76,6 +77,7 @@ public ColumnPruner() { * @param pactx * the current parse context */ + @Override public ParseContext transform(ParseContext pactx) throws SemanticException { pGraphContext = pactx; opToParseCtxMap = pGraphContext.getOpParseCtx(); @@ -120,6 +122,9 @@ public ParseContext transform(ParseContext pactx) throws SemanticException { opRules.put(new RuleRegExp("R11", ScriptOperator.getOperatorName() + "%"), ColumnPrunerProcFactory.getScriptProc()); + opRules.put(new RuleRegExp("R12", + LimitOperator.getOperatorName() + "%"), + ColumnPrunerProcFactory.getLimitProc()); // The dispatcher fires the processor corresponding to the closest matching // rule and passes the context along Dispatcher disp = new DefaultRuleDispatcher(ColumnPrunerProcFactory diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java index 0a9c6d2..ba28bc7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hive.ql.exec.JoinOperator; import org.apache.hadoop.hive.ql.exec.LateralViewForwardOperator; import org.apache.hadoop.hive.ql.exec.LateralViewJoinOperator; +import org.apache.hadoop.hive.ql.exec.LimitOperator; import org.apache.hadoop.hive.ql.exec.MapJoinOperator; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.OperatorFactory; @@ -215,6 +216,24 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx ctx, } } + public static class ColumnPrunerLimitProc extends ColumnPrunerDefaultProc { + + @Override + public Object process(Node nd, Stack stack, NodeProcessorCtx ctx, + Object... nodeOutputs) throws SemanticException { + super.process(nd, stack, ctx, nodeOutputs); + List cols = ((ColumnPrunerProcCtx)ctx).getPrunedColLists().get(nd); + if (null != cols) { + pruneOperator(ctx, (LimitOperator) nd, cols); + } + return null; + } + } + + public static ColumnPrunerLimitProc getLimitProc() { + return new ColumnPrunerLimitProc(); + } + public static ColumnPrunerScriptProc getScriptProc() { return new ColumnPrunerScriptProc(); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java index b323cb5..d175e64 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java @@ -29,9 +29,11 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.exec.FileSinkOperator; import org.apache.hadoop.hive.ql.exec.FilterOperator; +import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.exec.GroupByOperator; import org.apache.hadoop.hive.ql.exec.JoinOperator; import org.apache.hadoop.hive.ql.exec.Operator; @@ -57,15 +59,16 @@ import org.apache.hadoop.hive.ql.plan.GroupByDesc; import org.apache.hadoop.hive.ql.plan.JoinCondDesc; import org.apache.hadoop.hive.ql.plan.JoinDesc; -import org.apache.hadoop.hive.ql.plan.PlanUtils; import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc; import org.apache.hadoop.hive.ql.plan.TableScanDesc; import org.apache.hadoop.hive.ql.udf.UDFType; import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseCompare; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotNull; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNull; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr; import org.apache.hadoop.hive.serde.serdeConstants; @@ -163,6 +166,10 @@ private static ExprNodeConstantDesc typeCast(ExprNodeDesc desc, TypeInfo ti) { } LOG.debug("Casting " + desc + " to type " + ti); ExprNodeConstantDesc c = (ExprNodeConstantDesc) desc; + if (null != c.getFoldedFromVal() && priti.getTypeName().equals(serdeConstants.STRING_TYPE_NAME)) { + // avoid double casting to preserve original string representation of constant. + return new ExprNodeConstantDesc(c.getFoldedFromVal()); + } ObjectInspector origOI = TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(desc.getTypeInfo()); ObjectInspector oi = @@ -187,6 +194,14 @@ private static ExprNodeConstantDesc typeCast(ExprNodeDesc desc, TypeInfo ti) { return new ExprNodeConstantDesc(ti, convObj); } + public static ExprNodeDesc foldExpr(ExprNodeGenericFuncDesc funcDesc) { + + GenericUDF udf = funcDesc.getGenericUDF(); + if (!isDeterministicUdf(udf)) { + return funcDesc; + } + return evaluateFunction(funcDesc.getGenericUDF(),funcDesc.getChildren(), funcDesc.getChildren()); + } /** * Fold input expression desc. * @@ -278,7 +293,7 @@ private static boolean isDeterministicUdf(GenericUDF udf) { (UDF) Class.forName(bridge.getUdfClassName(), true, Utilities.getSessionSpecifiedClassLoader()) .newInstance(); files = udfInternal.getRequiredFiles(); - jars = udf.getRequiredJars(); + jars = udfInternal.getRequiredJars(); } catch (Exception e) { LOG.error("The UDF implementation class '" + udfClassName + "' is not present in the class path"); @@ -308,15 +323,22 @@ private static void propagate(GenericUDF udf, List newExprs, RowRe if (udf instanceof GenericUDFOPEqual) { ExprNodeDesc lOperand = newExprs.get(0); ExprNodeDesc rOperand = newExprs.get(1); - ExprNodeColumnDesc c; ExprNodeConstantDesc v; - if (lOperand instanceof ExprNodeColumnDesc && rOperand instanceof ExprNodeConstantDesc) { - c = (ExprNodeColumnDesc) lOperand; - v = (ExprNodeConstantDesc) rOperand; - } else if (rOperand instanceof ExprNodeColumnDesc && lOperand instanceof ExprNodeConstantDesc) { - c = (ExprNodeColumnDesc) rOperand; + if (lOperand instanceof ExprNodeConstantDesc) { v = (ExprNodeConstantDesc) lOperand; + } else if (rOperand instanceof ExprNodeConstantDesc) { + v = (ExprNodeConstantDesc) rOperand; } else { + // we need a constant on one side. + return; + } + // If both sides are constants, there is nothing to propagate + ExprNodeColumnDesc c = getColumnExpr(lOperand); + if (null == c) { + c = getColumnExpr(rOperand); + } + if (null == c) { + // we need a column expression on other side. return; } ColumnInfo ci = resolveColumn(rr, c); @@ -342,21 +364,42 @@ private static void propagate(GenericUDF udf, List newExprs, RowRe } } + private static ExprNodeColumnDesc getColumnExpr(ExprNodeDesc expr) { + while (FunctionRegistry.isOpCast(expr)) { + expr = expr.getChildren().get(0); + } + return (expr instanceof ExprNodeColumnDesc) ? (ExprNodeColumnDesc)expr : null; + } + private static ExprNodeDesc shortcutFunction(GenericUDF udf, List newExprs) { if (udf instanceof GenericUDFOPAnd) { for (int i = 0; i < 2; i++) { ExprNodeDesc childExpr = newExprs.get(i); + ExprNodeDesc other = newExprs.get(Math.abs(i - 1)); if (childExpr instanceof ExprNodeConstantDesc) { ExprNodeConstantDesc c = (ExprNodeConstantDesc) childExpr; if (Boolean.TRUE.equals(c.getValue())) { // if true, prune it - return newExprs.get(Math.abs(i - 1)); + return other; } else { // if false return false return childExpr; } + } else // Try to fold (key = 86) and (key is not null) to (key = 86) + if (childExpr instanceof ExprNodeGenericFuncDesc && + ((ExprNodeGenericFuncDesc)childExpr).getGenericUDF() instanceof GenericUDFOPNotNull && + childExpr.getChildren().get(0) instanceof ExprNodeColumnDesc && other instanceof ExprNodeGenericFuncDesc + && ((ExprNodeGenericFuncDesc)other).getGenericUDF() instanceof GenericUDFBaseCompare + && other.getChildren().size() == 2) { + ExprNodeColumnDesc colDesc = getColumnExpr(other.getChildren().get(0)); + if (null == colDesc) { + colDesc = getColumnExpr(other.getChildren().get(1)); + } + if (null != colDesc && colDesc.isSame(childExpr.getChildren().get(0))) { + return other; + } } } } @@ -470,6 +513,16 @@ private static ExprNodeDesc evaluateFunction(GenericUDF udf, List // FIXME: add null support. return null; + } else if (desc instanceof ExprNodeGenericFuncDesc) { + ExprNodeDesc evaluatedFn = foldExpr((ExprNodeGenericFuncDesc)desc); + if (null == evaluatedFn || !(evaluatedFn instanceof ExprNodeConstantDesc)) { + return null; + } + ExprNodeConstantDesc constant = (ExprNodeConstantDesc) evaluatedFn; + Object writableValue = PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector( + (PrimitiveTypeInfo) constant.getTypeInfo()).getPrimitiveWritableObject(constant.getValue()); + arguments[i] = new DeferredJavaObject(writableValue); + argois[i] = ObjectInspectorUtils.getConstantObjectInspector(constant.getWritableObjectInspector(), writableValue); } else { return null; } @@ -502,7 +555,12 @@ private static ExprNodeDesc evaluateFunction(GenericUDF udf, List LOG.error("Unable to evaluate " + udf + ". Return value unrecoginizable."); return null; } - return new ExprNodeConstantDesc(o); + String constStr = null; + if(arguments.length == 1 && FunctionRegistry.isOpCast(udf)) { + // remember original string representation of constant. + constStr = arguments[0].get().toString(); + } + return new ExprNodeConstantDesc(o).setFoldedFromVal(constStr); } catch (HiveException e) { LOG.error("Evaluation function " + udf.getClass() + " failed in Constant Propagatation Optimizer."); @@ -675,6 +733,21 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx ctx, Object.. if (colList != null) { for (int i = 0; i < colList.size(); i++) { ExprNodeDesc newCol = foldExpr(colList.get(i), constants, cppCtx, op, 0, false); + if (!(colList.get(i) instanceof ExprNodeConstantDesc) && newCol instanceof ExprNodeConstantDesc) { + // Lets try to store original column name, if this column got folded + // This is useful for optimizations like GroupByOptimizer + String colName = colList.get(i).getExprString(); + if (HiveConf.getPositionFromInternalName(colName) == -1) { + // if its not an internal name, this is what we want. + ((ExprNodeConstantDesc)newCol).setFoldedFromCol(colName); + } else { + // If it was internal column, lets try to get name from columnExprMap + ExprNodeDesc desc = columnExprMap.get(colName); + if (desc instanceof ExprNodeConstantDesc) { + ((ExprNodeConstantDesc)newCol).setFoldedFromCol(((ExprNodeConstantDesc)desc).getFoldedFromCol()); + } + } + } colList.set(i, newCol); if (columnExprMap != null) { columnExprMap.put(columnNames.get(i), newCol); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java index 7a3280c..bedc3ac 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java @@ -85,13 +85,18 @@ JoinOperator joinOp = (JoinOperator) nd; - if (!context.conf.getBoolVar(HiveConf.ConfVars.HIVECONVERTJOIN) - && !(context.conf.getBoolVar(HiveConf.ConfVars.HIVE_AUTO_SORTMERGE_JOIN))) { + TezBucketJoinProcCtx tezBucketJoinProcCtx = new TezBucketJoinProcCtx(context.conf); + if (!context.conf.getBoolVar(HiveConf.ConfVars.HIVECONVERTJOIN)) { // we are just converting to a common merge join operator. The shuffle // join in map-reduce case. - int pos = 0; // it doesn't matter which position we use in this case. - convertJoinSMBJoin(joinOp, context, pos, 0, false, false); - return null; + Object retval = checkAndConvertSMBJoin(context, joinOp, tezBucketJoinProcCtx); + if (retval == null) { + return retval; + } else { + int pos = 0; // it doesn't matter which position we use in this case. + convertJoinSMBJoin(joinOp, context, pos, 0, false, false); + return null; + } } // if we have traits, and table info is present in the traits, we know the @@ -99,7 +104,6 @@ // reducers from the parent operators. int numBuckets = -1; int estimatedBuckets = -1; - TezBucketJoinProcCtx tezBucketJoinProcCtx = new TezBucketJoinProcCtx(context.conf); if (context.conf.getBoolVar(HiveConf.ConfVars.HIVE_CONVERT_JOIN_BUCKET_MAPJOIN_TEZ)) { for (OperatorparentOp : joinOp.getParentOperators()) { if (parentOp.getOpTraits().getNumBuckets() > 0) { @@ -126,53 +130,15 @@ LOG.info("Estimated number of buckets " + numBuckets); int mapJoinConversionPos = getMapJoinConversionPos(joinOp, context, numBuckets); if (mapJoinConversionPos < 0) { - // we cannot convert to bucket map join, we cannot convert to - // map join either based on the size. Check if we can convert to SMB join. - if (context.conf.getBoolVar(HiveConf.ConfVars.HIVE_AUTO_SORTMERGE_JOIN) == false) { - convertJoinSMBJoin(joinOp, context, 0, 0, false, false); - return null; - } - Class bigTableMatcherClass = null; - try { - bigTableMatcherClass = - (Class) (Class.forName(HiveConf.getVar( - context.parseContext.getConf(), - HiveConf.ConfVars.HIVE_AUTO_SORTMERGE_JOIN_BIGTABLE_SELECTOR))); - } catch (ClassNotFoundException e) { - throw new SemanticException(e.getMessage()); - } - - BigTableSelectorForAutoSMJ bigTableMatcher = - ReflectionUtils.newInstance(bigTableMatcherClass, null); - JoinDesc joinDesc = joinOp.getConf(); - JoinCondDesc[] joinCondns = joinDesc.getConds(); - Set joinCandidates = MapJoinProcessor.getBigTableCandidates(joinCondns); - if (joinCandidates.isEmpty()) { - // This is a full outer join. This can never be a map-join - // of any type. So return false. - return false; - } - mapJoinConversionPos = - bigTableMatcher.getBigTablePosition(context.parseContext, joinOp, joinCandidates); - if (mapJoinConversionPos < 0) { - // contains aliases from sub-query - // we are just converting to a common merge join operator. The shuffle - // join in map-reduce case. - int pos = 0; // it doesn't matter which position we use in this case. - convertJoinSMBJoin(joinOp, context, pos, 0, false, false); - return null; - } - - if (checkConvertJoinSMBJoin(joinOp, context, mapJoinConversionPos, tezBucketJoinProcCtx)) { - convertJoinSMBJoin(joinOp, context, mapJoinConversionPos, - tezBucketJoinProcCtx.getNumBuckets(), tezBucketJoinProcCtx.isSubQuery(), true); + Object retval = checkAndConvertSMBJoin(context, joinOp, tezBucketJoinProcCtx); + if (retval == null) { + return retval; } else { - // we are just converting to a common merge join operator. The shuffle - // join in map-reduce case. - int pos = 0; // it doesn't matter which position we use in this case. - convertJoinSMBJoin(joinOp, context, pos, 0, false, false); + // only case is full outer join with SMB enabled which is not possible. Convert to regular + // join. + convertJoinSMBJoin(joinOp, context, 0, 0, false, false); + return null; } - return null; } if (numBuckets > 1) { @@ -206,6 +172,57 @@ return null; } + private Object checkAndConvertSMBJoin(OptimizeTezProcContext context, JoinOperator joinOp, + TezBucketJoinProcCtx tezBucketJoinProcCtx) throws SemanticException { + // we cannot convert to bucket map join, we cannot convert to + // map join either based on the size. Check if we can convert to SMB join. + if (context.conf.getBoolVar(HiveConf.ConfVars.HIVE_AUTO_SORTMERGE_JOIN) == false) { + convertJoinSMBJoin(joinOp, context, 0, 0, false, false); + return null; + } + Class bigTableMatcherClass = null; + try { + bigTableMatcherClass = + (Class) (Class.forName(HiveConf.getVar( + context.parseContext.getConf(), + HiveConf.ConfVars.HIVE_AUTO_SORTMERGE_JOIN_BIGTABLE_SELECTOR))); + } catch (ClassNotFoundException e) { + throw new SemanticException(e.getMessage()); + } + + BigTableSelectorForAutoSMJ bigTableMatcher = + ReflectionUtils.newInstance(bigTableMatcherClass, null); + JoinDesc joinDesc = joinOp.getConf(); + JoinCondDesc[] joinCondns = joinDesc.getConds(); + Set joinCandidates = MapJoinProcessor.getBigTableCandidates(joinCondns); + if (joinCandidates.isEmpty()) { + // This is a full outer join. This can never be a map-join + // of any type. So return false. + return false; + } + int mapJoinConversionPos = + bigTableMatcher.getBigTablePosition(context.parseContext, joinOp, joinCandidates); + if (mapJoinConversionPos < 0) { + // contains aliases from sub-query + // we are just converting to a common merge join operator. The shuffle + // join in map-reduce case. + int pos = 0; // it doesn't matter which position we use in this case. + convertJoinSMBJoin(joinOp, context, pos, 0, false, false); + return null; + } + + if (checkConvertJoinSMBJoin(joinOp, context, mapJoinConversionPos, tezBucketJoinProcCtx)) { + convertJoinSMBJoin(joinOp, context, mapJoinConversionPos, + tezBucketJoinProcCtx.getNumBuckets(), tezBucketJoinProcCtx.isSubQuery(), true); + } else { + // we are just converting to a common merge join operator. The shuffle + // join in map-reduce case. + int pos = 0; // it doesn't matter which position we use in this case. + convertJoinSMBJoin(joinOp, context, pos, 0, false, false); + } + return null; +} + // replaces the join operator with a new CommonJoinOperator, removes the // parent reduce sinks private void convertJoinSMBJoin(JoinOperator joinOp, OptimizeTezProcContext context, @@ -228,7 +245,7 @@ private void convertJoinSMBJoin(JoinOperator joinOp, OptimizeTezProcContext cont @SuppressWarnings("unchecked") CommonMergeJoinOperator mergeJoinOp = (CommonMergeJoinOperator) OperatorFactory.get(new CommonMergeJoinDesc(numBuckets, - isSubQuery, mapJoinConversionPos, mapJoinDesc)); + isSubQuery, mapJoinConversionPos, mapJoinDesc), joinOp.getSchema()); OpTraits opTraits = new OpTraits(joinOp.getOpTraits().getBucketColNames(), numBuckets, joinOp.getOpTraits() .getSortCols()); @@ -630,7 +647,6 @@ private boolean hasDynamicPartitionBroadcast(Operator parent) { hasDynamicPartitionPruning = true; break; } - if (op instanceof ReduceSinkOperator || op instanceof FileSinkOperator) { // crossing reduce sink or file sink means the pruning isn't for this parent. break; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java index d06522f..1d18e0c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java @@ -332,18 +332,26 @@ protected GroupByOptimizerSortMatch checkSortGroupBy(Stack stack, continue; } - ExprNodeDesc selectColList = selectDesc.getColList().get(pos); - if (selectColList instanceof ExprNodeColumnDesc) { + ExprNodeDesc selectCol = selectDesc.getColList().get(pos); + if (selectCol instanceof ExprNodeColumnDesc) { String newValue = - tableColsMapping.get(((ExprNodeColumnDesc) selectColList).getColumn()); + tableColsMapping.get(((ExprNodeColumnDesc) selectCol).getColumn()); tableColsMapping.put(outputColumnName, newValue); } else { tableColsMapping.remove(outputColumnName); - if ((selectColList instanceof ExprNodeConstantDesc) || - (selectColList instanceof ExprNodeNullDesc)) { + if (selectCol instanceof ExprNodeNullDesc) { newConstantCols.add(outputColumnName); } + if (selectCol instanceof ExprNodeConstantDesc) { + // Lets see if this constant was folded because of optimization. + String origCol = ((ExprNodeConstantDesc) selectCol).getFoldedFromCol(); + if (origCol != null) { + tableColsMapping.put(outputColumnName, origCol); + } else { + newConstantCols.add(outputColumnName); + } + } } } @@ -351,7 +359,6 @@ protected GroupByOptimizerSortMatch checkSortGroupBy(Stack stack, } } - boolean sortGroupBy = true; // compute groupby columns from groupby keys List groupByCols = new ArrayList(); // If the group by expression is anything other than a list of columns, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MergeJoinProc.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MergeJoinProc.java index 8516643..e644481 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MergeJoinProc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MergeJoinProc.java @@ -1,7 +1,5 @@ package org.apache.hadoop.hive.ql.optimizer; -import java.util.HashMap; -import java.util.Map; import java.util.Stack; import org.apache.hadoop.hive.ql.exec.CommonMergeJoinOperator; @@ -22,21 +20,6 @@ import org.apache.hadoop.hive.ql.plan.TezWork.VertexType; public class MergeJoinProc implements NodeProcessor { - - public Operator getLeafOperator(Operator op) { - for (Operator childOp : op.getChildOperators()) { - // FileSink or ReduceSink operators are used to create vertices. See - // TezCompiler. - if ((childOp instanceof ReduceSinkOperator) || (childOp instanceof FileSinkOperator)) { - return childOp; - } else { - return getLeafOperator(childOp); - } - } - - return null; - } - @Override public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Object... nodeOutputs) @@ -60,13 +43,13 @@ // merge work already exists for this merge join operator, add the dummy store work to the // merge work. Else create a merge work, add above work to the merge work MergeJoinWork mergeWork = null; - if (context.opMergeJoinWorkMap.containsKey(getLeafOperator(mergeJoinOp))) { + if (context.opMergeJoinWorkMap.containsKey(mergeJoinOp)) { // we already have the merge work corresponding to this merge join operator - mergeWork = context.opMergeJoinWorkMap.get(getLeafOperator(mergeJoinOp)); + mergeWork = context.opMergeJoinWorkMap.get(mergeJoinOp); } else { mergeWork = new MergeJoinWork(); tezWork.add(mergeWork); - context.opMergeJoinWorkMap.put(getLeafOperator(mergeJoinOp), mergeWork); + context.opMergeJoinWorkMap.put(mergeJoinOp, mergeWork); } mergeWork.setMergeJoinOperator(mergeJoinOp); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java index da764cf..778a68a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java @@ -21,6 +21,8 @@ import java.util.ArrayList; import java.util.List; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.optimizer.correlation.CorrelationOptimizer; import org.apache.hadoop.hive.ql.optimizer.correlation.ReduceSinkDeDuplication; @@ -44,6 +46,7 @@ public class Optimizer { private ParseContext pctx; private List transformations; + private static final Log LOG = LogFactory.getLog(Optimizer.class.getName()); /** * Create the list of transformations. @@ -61,9 +64,19 @@ public void initialize(HiveConf hiveConf) { // Add the transformation that computes the lineage information. transformations.add(new Generator()); if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTPPD)) { - transformations.add(new PredicateTransitivePropagate()); + transformations.add(new PredicateTransitivePropagate()); + if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTCONSTANTPROPAGATION)) { + transformations.add(new ConstantPropagate()); + } transformations.add(new SyntheticJoinPredicate()); transformations.add(new PredicatePushDown()); + } + if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTCONSTANTPROPAGATION)) { + // We run constant propagation twice because after predicate pushdown, filter expressions + // are combined and may become eligible for reduction (like is not null filter). + transformations.add(new ConstantPropagate()); + } + if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTPPD)) { transformations.add(new PartitionPruner()); transformations.add(new PartitionConditionRemover()); if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTLISTBUCKETING)) { @@ -71,16 +84,18 @@ public void initialize(HiveConf hiveConf) { transformations.add(new ListBucketingPruner()); } } + if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTGROUPBY) || HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_MAP_GROUPBY_SORT)) { transformations.add(new GroupByOptimizer()); } - if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTCONSTANTPROPAGATION)) { - transformations.add(new ConstantPropagate()); - } transformations.add(new ColumnPruner()); if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_OPTIMIZE_SKEWJOIN_COMPILETIME)) { - transformations.add(new SkewJoinOptimizer()); + if (!isTezExecEngine) { + transformations.add(new SkewJoinOptimizer()); + } else { + LOG.warn("Skew join is currently not supported in tez! Disabling the skew join optimization."); + } } if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTGBYUSINGINDEX)) { transformations.add(new RewriteGBUsingIndex()); @@ -140,7 +155,9 @@ public void initialize(HiveConf hiveConf) { transformations.add(new AnnotateWithOpTraits()); } - transformations.add(new SimpleFetchOptimizer()); // must be called last + if (!HiveConf.getVar(hiveConf, HiveConf.ConfVars.HIVEFETCHTASKCONVERSION).equals("none")) { + transformations.add(new SimpleFetchOptimizer()); // must be called last + } if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEFETCHTASKAGGR)) { transformations.add(new SimpleFetchAggregation()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java index 65fb66e..ed6f713 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.optimizer; import java.util.ArrayList; +import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -55,6 +56,8 @@ import org.apache.hadoop.hive.ql.plan.TezWork.VertexType; import org.apache.hadoop.hive.ql.stats.StatsUtils; +import static org.apache.hadoop.hive.ql.plan.ReduceSinkDesc.ReducerTraits.FIXED; + public class ReduceSinkMapJoinProc implements NodeProcessor { protected transient Log LOG = LogFactory.getLog(this.getClass().getName()); @@ -167,7 +170,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procContext, if (joinConf.isBucketMapJoin()) { // disable auto parallelism for bucket map joins - parentRS.getConf().setAutoParallel(false); + parentRS.getConf().setReducerTraits(EnumSet.of(FIXED)); numBuckets = (Integer) joinConf.getBigTableBucketNumMapping().values().toArray()[0]; if (joinConf.getCustomBucketMapJoin()) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SetReducerParallelism.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SetReducerParallelism.java index 625af85..fef2c29 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SetReducerParallelism.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SetReducerParallelism.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.ql.optimizer; +import java.util.Collection; +import java.util.EnumSet; import java.util.Stack; import org.apache.commons.logging.Log; @@ -31,9 +33,13 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.parse.OptimizeTezProcContext; import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.ExprNodeDesc.ExprNodeDescEqualityWrapper; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc; +import static org.apache.hadoop.hive.ql.plan.ReduceSinkDesc.ReducerTraits.AUTOPARALLEL; +import static org.apache.hadoop.hive.ql.plan.ReduceSinkDesc.ReducerTraits.UNIFORM; + /** * SetReducerParallelism determines how many reducers should * be run for a given reduce sink. @@ -86,7 +92,14 @@ public Object process(Node nd, Stack stack, maxReducers, false); LOG.info("Set parallelism for reduce sink "+sink+" to: "+numReducers); desc.setNumReducers(numReducers); - desc.setAutoParallel(true); + + final Collection keyCols = ExprNodeDescEqualityWrapper.transform(desc.getKeyCols()); + final Collection partCols = ExprNodeDescEqualityWrapper.transform(desc.getPartitionCols()); + if (keyCols != null && keyCols.equals(partCols)) { + desc.setReducerTraits(EnumSet.of(UNIFORM, AUTOPARALLEL)); + } else { + desc.setReducerTraits(EnumSet.of(AUTOPARALLEL)); + } } } else { LOG.info("Number of reducers determined to be: "+desc.getNumReducers()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java index dee7d7e..3097385 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java @@ -112,7 +112,7 @@ private static void doSemanticAnalysis(SemanticAnalyzer sem, ((SemanticAnalyzer) sem).initParseCtx(subPCtx); LOG.info("Starting Sub-query Semantic Analysis"); - sem.doPhase1(child, qb, sem.initPhase1Ctx()); + sem.doPhase1(child, qb, sem.initPhase1Ctx(), null); LOG.info("Completed phase 1 of Sub-query Semantic Analysis"); sem.getMetaData(qb); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveOptiqUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveOptiqUtil.java index 7c2b0cd..a14770e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveOptiqUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveOptiqUtil.java @@ -77,7 +77,8 @@ public static boolean validateASTForCBO(ASTNode ast) { String astTree = ast.toStringTree(); - String[] tokens = { "TOK_CHARSETLITERAL" }; + // if any of following tokens are present in AST, bail out + String[] tokens = { "TOK_CHARSETLITERAL","TOK_TABLESPLITSAMPLE" }; for (String token : tokens) { if (astTree.contains(token)) { return false; @@ -241,7 +242,7 @@ public static RexNode projectNonColumnEquiConditions(ProjectFactory factory, Rel *

* JoinPredicateInfo:
* 1. preserves the order of conjuctive elements for - * equi-join(m_equiJoinPredicateElements)
+ * equi-join(equiJoinPredicateElements)
* 2. Stores set of projection indexes from left and right child which is part * of equi join keys; the indexes are both in child and Join node schema.
* 3. Keeps a map of projection indexes that are part of join keys to list of @@ -275,19 +276,19 @@ public JoinPredicateInfo(List nonEquiJoinPredicateElement } public List getNonEquiJoinPredicateElements() { - return nonEquiJoinPredicateElements; + return this.nonEquiJoinPredicateElements; } public List getEquiJoinPredicateElements() { - return equiJoinPredicateElements; + return this.equiJoinPredicateElements; } public Set getProjsFromLeftPartOfJoinKeysInChildSchema() { - return projsFromLeftPartOfJoinKeysInChildSchema; + return this.projsFromLeftPartOfJoinKeysInChildSchema; } public Set getProjsFromRightPartOfJoinKeysInChildSchema() { - return projsFromRightPartOfJoinKeysInChildSchema; + return this.projsFromRightPartOfJoinKeysInChildSchema; } /** @@ -296,15 +297,15 @@ public JoinPredicateInfo(List nonEquiJoinPredicateElement * schema. */ public Set getProjsFromLeftPartOfJoinKeysInJoinSchema() { - return projsFromLeftPartOfJoinKeysInChildSchema; + return this.projsFromLeftPartOfJoinKeysInChildSchema; } public Set getProjsFromRightPartOfJoinKeysInJoinSchema() { - return projsFromRightPartOfJoinKeysInJoinSchema; + return this.projsFromRightPartOfJoinKeysInJoinSchema; } public Map> getMapOfProjIndxToLeafPInfo() { - return mapOfProjIndxInJoinSchemaToLeafPInfo; + return this.mapOfProjIndxInJoinSchemaToLeafPInfo; } public static JoinPredicateInfo constructJoinPredicateInfo(HiveJoinRel j) { @@ -336,7 +337,7 @@ public static JoinPredicateInfo constructJoinPredicateInfo(HiveJoinRel j, RexNod jlpi = JoinLeafPredicateInfo.constructJoinLeafPredicateInfo(j, ce); // 2.2 Classify leaf predicate as Equi vs Non Equi - if (jlpi.m_comparisonType.equals(SqlKind.EQUALS)) { + if (jlpi.comparisonType.equals(SqlKind.EQUALS)) { equiLPIList.add(jlpi); } else { nonEquiLPIList.add(jlpi); @@ -398,38 +399,38 @@ public static JoinPredicateInfo constructJoinPredicateInfo(HiveJoinRel j, RexNod * of equi join keys; the indexes are both in child and Join node schema.
*/ public static class JoinLeafPredicateInfo { - private final SqlKind m_comparisonType; - private final ImmutableList m_joinKeyExprsFromLeft; - private final ImmutableList m_joinKeyExprsFromRight; - private final ImmutableSet m_projsFromLeftPartOfJoinKeysInChildSchema; - private final ImmutableSet m_projsFromRightPartOfJoinKeysInChildSchema; - private final ImmutableSet m_projsFromRightPartOfJoinKeysInJoinSchema; + private final SqlKind comparisonType; + private final ImmutableList joinKeyExprsFromLeft; + private final ImmutableList joinKeyExprsFromRight; + private final ImmutableSet projsFromLeftPartOfJoinKeysInChildSchema; + private final ImmutableSet projsFromRightPartOfJoinKeysInChildSchema; + private final ImmutableSet projsFromRightPartOfJoinKeysInJoinSchema; public JoinLeafPredicateInfo(SqlKind comparisonType, List joinKeyExprsFromLeft, List joinKeyExprsFromRight, Set projsFromLeftPartOfJoinKeysInChildSchema, Set projsFromRightPartOfJoinKeysInChildSchema, Set projsFromRightPartOfJoinKeysInJoinSchema) { - m_comparisonType = comparisonType; - m_joinKeyExprsFromLeft = ImmutableList.copyOf(joinKeyExprsFromLeft); - m_joinKeyExprsFromRight = ImmutableList.copyOf(joinKeyExprsFromRight); - m_projsFromLeftPartOfJoinKeysInChildSchema = ImmutableSet + this.comparisonType = comparisonType; + this.joinKeyExprsFromLeft = ImmutableList.copyOf(joinKeyExprsFromLeft); + this.joinKeyExprsFromRight = ImmutableList.copyOf(joinKeyExprsFromRight); + this.projsFromLeftPartOfJoinKeysInChildSchema = ImmutableSet .copyOf(projsFromLeftPartOfJoinKeysInChildSchema); - m_projsFromRightPartOfJoinKeysInChildSchema = ImmutableSet + this.projsFromRightPartOfJoinKeysInChildSchema = ImmutableSet .copyOf(projsFromRightPartOfJoinKeysInChildSchema); - m_projsFromRightPartOfJoinKeysInJoinSchema = ImmutableSet + this.projsFromRightPartOfJoinKeysInJoinSchema = ImmutableSet .copyOf(projsFromRightPartOfJoinKeysInJoinSchema); } public List getJoinKeyExprsFromLeft() { - return m_joinKeyExprsFromLeft; + return this.joinKeyExprsFromLeft; } public List getJoinKeyExprsFromRight() { - return m_joinKeyExprsFromRight; + return this.joinKeyExprsFromRight; } public Set getProjsFromLeftPartOfJoinKeysInChildSchema() { - return m_projsFromLeftPartOfJoinKeysInChildSchema; + return this.projsFromLeftPartOfJoinKeysInChildSchema; } /** @@ -438,15 +439,15 @@ public JoinLeafPredicateInfo(SqlKind comparisonType, List joinKeyExprsF * schema. */ public Set getProjsFromLeftPartOfJoinKeysInJoinSchema() { - return m_projsFromLeftPartOfJoinKeysInChildSchema; + return this.projsFromLeftPartOfJoinKeysInChildSchema; } public Set getProjsFromRightPartOfJoinKeysInChildSchema() { - return m_projsFromRightPartOfJoinKeysInChildSchema; + return this.projsFromRightPartOfJoinKeysInChildSchema; } public Set getProjsFromRightPartOfJoinKeysInJoinSchema() { - return m_projsFromRightPartOfJoinKeysInJoinSchema; + return this.projsFromRightPartOfJoinKeysInJoinSchema; } private static JoinLeafPredicateInfo constructJoinLeafPredicateInfo(HiveJoinRel j, RexNode pe) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushFilterPastJoinRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushFilterPastJoinRule.java index da0f7a4..ec72047 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushFilterPastJoinRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushFilterPastJoinRule.java @@ -17,274 +17,135 @@ */ package org.apache.hadoop.hive.ql.optimizer.optiq.rules; -import java.util.ArrayList; import java.util.BitSet; import java.util.List; import java.util.ListIterator; import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveFilterRel; -import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveJoinRel; -import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveRel; +import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveProjectRel; import org.eigenbase.rel.FilterRelBase; import org.eigenbase.rel.JoinRelBase; import org.eigenbase.rel.JoinRelType; -import org.eigenbase.rel.RelNode; +import org.eigenbase.rel.RelFactories; +import org.eigenbase.rel.rules.PushFilterPastJoinRule; import org.eigenbase.relopt.RelOptRule; import org.eigenbase.relopt.RelOptRuleCall; import org.eigenbase.relopt.RelOptRuleOperand; -import org.eigenbase.relopt.RelOptUtil; import org.eigenbase.relopt.RelOptUtil.InputFinder; -import org.eigenbase.rex.RexBuilder; import org.eigenbase.rex.RexCall; import org.eigenbase.rex.RexNode; -import org.eigenbase.rex.RexUtil; import org.eigenbase.sql.SqlKind; -import org.eigenbase.util.Holder; -public abstract class HivePushFilterPastJoinRule extends RelOptRule { - - public static final HivePushFilterPastJoinRule FILTER_ON_JOIN = new HivePushFilterPastJoinRule( - operand(FilterRelBase.class, operand(HiveJoinRel.class, any())), - "HivePushFilterPastJoinRule:filter", true) { - @Override - public void onMatch(RelOptRuleCall call) { - HiveFilterRel filter = call.rel(0); - HiveJoinRel join = call.rel(1); - perform(call, filter, join); - } - }; - - public static final HivePushFilterPastJoinRule JOIN = new HivePushFilterPastJoinRule( - operand(HiveJoinRel.class, any()), "HivePushFilterPastJoinRule:no-filter", false) { - @Override - public void onMatch(RelOptRuleCall call) { - HiveJoinRel join = call.rel(0); - perform(call, null, join); - } - }; - - /** Whether to try to strengthen join-type. */ - private final boolean smart; - - // ~ Constructors ----------------------------------------------------------- - - /** - * Creates a PushFilterPastJoinRule with an explicit root operand. - */ - private HivePushFilterPastJoinRule(RelOptRuleOperand operand, String id, boolean smart) { - super(operand, "PushFilterRule: " + id); - this.smart = smart; - } - - // ~ Methods ---------------------------------------------------------------- - - protected void perform(RelOptRuleCall call, FilterRelBase filter, - JoinRelBase join) { - final List joinFilters = RelOptUtil.conjunctions(join - .getCondition()); - - /* - * todo: hb 6/26/14 for left SemiJoin we cannot push predicates yet. The - * assertion that num(JoinRel columns) = num(leftSrc) + num(rightSrc) - * doesn't hold. So RelOptUtil.classifyFilters fails. - */ - if (((HiveJoinRel) join).isLeftSemiJoin()) { - return; - } - - if (filter == null) { - // There is only the joinRel - // make sure it does not match a cartesian product joinRel - // (with "true" condition) otherwise this rule will be applied - // again on the new cartesian product joinRel. - boolean onlyTrueFilter = true; - for (RexNode joinFilter : joinFilters) { - if (!joinFilter.isAlwaysTrue()) { - onlyTrueFilter = false; - break; - } - } - - if (onlyTrueFilter) { - return; - } - } - - final List aboveFilters = filter != null ? RelOptUtil - .conjunctions(filter.getCondition()) : new ArrayList(); - - List leftFilters = new ArrayList(); - List rightFilters = new ArrayList(); - int origJoinFiltersSz = joinFilters.size(); - - // TODO - add logic to derive additional filters. E.g., from - // (t1.a = 1 AND t2.a = 2) OR (t1.b = 3 AND t2.b = 4), you can - // derive table filters: - // (t1.a = 1 OR t1.b = 3) - // (t2.a = 2 OR t2.b = 4) - - // Try to push down above filters. These are typically where clause - // filters. They can be pushed down if they are not on the NULL - // generating side. - boolean filterPushed = false; - final Holder joinTypeHolder = Holder.of(join.getJoinType()); - if (RelOptUtil.classifyFilters(join, aboveFilters, - join.getJoinType(), true, !join.getJoinType().generatesNullsOnLeft(), !join.getJoinType() - .generatesNullsOnRight(), joinFilters, leftFilters, rightFilters, joinTypeHolder, smart)) { - filterPushed = true; - } - - /* - * Any predicates pushed down to joinFilters that aren't equality - * conditions: put them back as aboveFilters because Hive doesn't support - * not equi join conditions. - */ - ListIterator filterIter = joinFilters.listIterator(); - while (filterIter.hasNext()) { - RexNode exp = filterIter.next(); - if (exp instanceof RexCall) { - RexCall c = (RexCall) exp; - if (c.getOperator().getKind() == SqlKind.EQUALS) { - boolean validHiveJoinFilter = true; - for (RexNode rn : c.getOperands()) { - // NOTE: Hive dis-allows projections from both left & right side - // of join condition. Example: Hive disallows - // (r1.x=r2.x)=(r1.y=r2.y) on join condition. - if (filterRefersToBothSidesOfJoin(rn, join)) { - validHiveJoinFilter = false; - break; - } - } - if (validHiveJoinFilter) - continue; - } - } - aboveFilters.add(exp); - filterIter.remove(); - } - - /* - * if all pushed filters where put back then set filterPushed to false - */ - if (leftFilters.size() == 0 && rightFilters.size() == 0 - && joinFilters.size() == origJoinFiltersSz) { - filterPushed = false; - } - - // Try to push down filters in ON clause. A ON clause filter can only be - // pushed down if it does not affect the non-matching set, i.e. it is - // not on the side which is preserved. - if (RelOptUtil.classifyFilters(join, joinFilters, null, false, !join - .getJoinType().generatesNullsOnRight(), !join.getJoinType() - .generatesNullsOnLeft(), joinFilters, leftFilters, rightFilters, joinTypeHolder, false)) { - filterPushed = true; - } - - if (!filterPushed) { - return; - } - - /* - * Remove always true conditions that got pushed down. - */ - removeAlwaysTruePredicates(leftFilters); - removeAlwaysTruePredicates(rightFilters); - removeAlwaysTruePredicates(joinFilters); - - // create FilterRels on top of the children if any filters were - // pushed to them - RexBuilder rexBuilder = join.getCluster().getRexBuilder(); - RelNode leftRel = createFilterOnRel(rexBuilder, join.getLeft(), leftFilters); - RelNode rightRel = createFilterOnRel(rexBuilder, join.getRight(), - rightFilters); - - // create the new join node referencing the new children and - // containing its new join filters (if there are any) - RexNode joinFilter; - - if (joinFilters.size() == 0) { - // if nothing actually got pushed and there is nothing leftover, - // then this rule is a no-op - if (leftFilters.isEmpty() - && rightFilters.isEmpty() - && joinTypeHolder.get() == join.getJoinType()) { - return; - } - joinFilter = rexBuilder.makeLiteral(true); - } else { - joinFilter = RexUtil.composeConjunction(rexBuilder, joinFilters, true); - } - RelNode newJoinRel = HiveJoinRel.getJoin(join.getCluster(), leftRel, - rightRel, joinFilter, join.getJoinType(), false); - - // create a FilterRel on top of the join if needed - RelNode newRel = createFilterOnRel(rexBuilder, newJoinRel, aboveFilters); - - call.transformTo(newRel); - } - - /** - * If the filter list passed in is non-empty, creates a FilterRel on top of - * the existing RelNode; otherwise, just returns the RelNode - * - * @param rexBuilder - * rex builder - * @param rel - * the RelNode that the filter will be put on top of - * @param filters - * list of filters - * @return new RelNode or existing one if no filters - */ - private RelNode createFilterOnRel(RexBuilder rexBuilder, RelNode rel, - List filters) { - RexNode andFilters = RexUtil.composeConjunction(rexBuilder, filters, false); - if (andFilters.isAlwaysTrue()) { - return rel; - } - return new HiveFilterRel(rel.getCluster(), rel.getCluster().traitSetOf( - HiveRel.CONVENTION), rel, andFilters); - } - - private void removeAlwaysTruePredicates(List predicates) { - - ListIterator iter = predicates.listIterator(); - while (iter.hasNext()) { - RexNode exp = iter.next(); - if (isAlwaysTrue(exp)) { - iter.remove(); - } - } - } - - private boolean isAlwaysTrue(RexNode predicate) { - if (predicate instanceof RexCall) { - RexCall c = (RexCall) predicate; - if (c.getOperator().getKind() == SqlKind.EQUALS) { - return isAlwaysTrue(c.getOperands().get(0)) - && isAlwaysTrue(c.getOperands().get(1)); - } - } - return predicate.isAlwaysTrue(); - } - - private boolean filterRefersToBothSidesOfJoin(RexNode filter, JoinRelBase j) { - boolean refersToBothSides = false; - - int joinNoOfProjects = j.getRowType().getFieldCount(); - BitSet filterProjs = new BitSet(joinNoOfProjects); - BitSet allLeftProjs = new BitSet(joinNoOfProjects); - BitSet allRightProjs = new BitSet(joinNoOfProjects); - allLeftProjs.set(0, j.getInput(0).getRowType().getFieldCount(), true); - allRightProjs.set(j.getInput(0).getRowType().getFieldCount(), joinNoOfProjects, true); - - InputFinder inputFinder = new InputFinder(filterProjs); - filter.accept(inputFinder); - - if (allLeftProjs.intersects(filterProjs) && allRightProjs.intersects(filterProjs)) - refersToBothSides = true; - - return refersToBothSides; - } +public abstract class HivePushFilterPastJoinRule extends PushFilterPastJoinRule { + + public static final HivePushFilterPastJoinRule FILTER_ON_JOIN = new HivePushFilterIntoJoinRule(); + + public static final HivePushFilterPastJoinRule JOIN = new HivePushDownJoinConditionRule(); + + /** + * Creates a PushFilterPastJoinRule with an explicit root operand. + */ + protected HivePushFilterPastJoinRule(RelOptRuleOperand operand, String id, + boolean smart, RelFactories.FilterFactory filterFactory, + RelFactories.ProjectFactory projectFactory) { + super(operand, id, smart, filterFactory, projectFactory); + } + + /** + * Rule that tries to push filter expressions into a join condition and into + * the inputs of the join. + */ + public static class HivePushFilterIntoJoinRule extends + HivePushFilterPastJoinRule { + public HivePushFilterIntoJoinRule() { + super(RelOptRule.operand(FilterRelBase.class, + RelOptRule.operand(JoinRelBase.class, RelOptRule.any())), + "HivePushFilterPastJoinRule:filter", true, + HiveFilterRel.DEFAULT_FILTER_FACTORY, + HiveProjectRel.DEFAULT_PROJECT_FACTORY); + } + + @Override + public void onMatch(RelOptRuleCall call) { + FilterRelBase filter = call.rel(0); + JoinRelBase join = call.rel(1); + super.perform(call, filter, join); + } + } + + public static class HivePushDownJoinConditionRule extends + HivePushFilterPastJoinRule { + public HivePushDownJoinConditionRule() { + super(RelOptRule.operand(JoinRelBase.class, RelOptRule.any()), + "HivePushFilterPastJoinRule:no-filter", true, + HiveFilterRel.DEFAULT_FILTER_FACTORY, + HiveProjectRel.DEFAULT_PROJECT_FACTORY); + } + + @Override + public void onMatch(RelOptRuleCall call) { + JoinRelBase join = call.rel(0); + super.perform(call, null, join); + } + } + + /* + * Any predicates pushed down to joinFilters that aren't equality + * conditions: put them back as aboveFilters because Hive doesn't support + * not equi join conditions. + */ + @Override + protected void validateJoinFilters(List aboveFilters, + List joinFilters, JoinRelBase join, JoinRelType joinType) { + if (joinType.equals(JoinRelType.INNER)) { + ListIterator filterIter = joinFilters.listIterator(); + while (filterIter.hasNext()) { + RexNode exp = filterIter.next(); + if (exp instanceof RexCall) { + RexCall c = (RexCall) exp; + if (c.getOperator().getKind() == SqlKind.EQUALS) { + boolean validHiveJoinFilter = true; + for (RexNode rn : c.getOperands()) { + // NOTE: Hive dis-allows projections from both left + // & + // right side + // of join condition. Example: Hive disallows + // (r1.x=r2.x)=(r1.y=r2.y) on join condition. + if (filterRefersToBothSidesOfJoin(rn, join)) { + validHiveJoinFilter = false; + break; + } + } + if (validHiveJoinFilter) + continue; + } + } + aboveFilters.add(exp); + filterIter.remove(); + } + } + } + + private boolean filterRefersToBothSidesOfJoin(RexNode filter, JoinRelBase j) { + boolean refersToBothSides = false; + + int joinNoOfProjects = j.getRowType().getFieldCount(); + BitSet filterProjs = new BitSet(joinNoOfProjects); + BitSet allLeftProjs = new BitSet(joinNoOfProjects); + BitSet allRightProjs = new BitSet(joinNoOfProjects); + allLeftProjs.set(0, j.getInput(0).getRowType().getFieldCount(), true); + allRightProjs.set(j.getInput(0).getRowType().getFieldCount(), + joinNoOfProjects, true); + + InputFinder inputFinder = new InputFinder(filterProjs); + filter.accept(inputFinder); + + if (allLeftProjs.intersects(filterProjs) + && allRightProjs.intersects(filterProjs)) + refersToBothSides = true; + + return refersToBothSides; + } } // End PushFilterPastJoinRule.java diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/FilterSelectivityEstimator.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/FilterSelectivityEstimator.java index 7ebe652..ba07363 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/FilterSelectivityEstimator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/FilterSelectivityEstimator.java @@ -34,13 +34,13 @@ import org.eigenbase.sql.SqlKind; public class FilterSelectivityEstimator extends RexVisitorImpl { - private final RelNode m_childRel; - private final double m_childCardinality; + private final RelNode childRel; + private final double childCardinality; protected FilterSelectivityEstimator(RelNode childRel) { super(true); - m_childRel = childRel; - m_childCardinality = RelMetadataQuery.getRowCount(m_childRel); + this.childRel = childRel; + this.childCardinality = RelMetadataQuery.getRowCount(childRel); } public Double estimateSelectivity(RexNode predicate) { @@ -53,11 +53,10 @@ public Double visitCall(RexCall call) { } /* - * Ignore any predicates on partition columns - * because we have already accounted for these in - * the Table row count. + * Ignore any predicates on partition columns because we have already + * accounted for these in the Table row count. */ - if (isPartitionPredicate(call, m_childRel)) { + if (isPartitionPredicate(call, this.childRel)) { return 1.0; } @@ -77,6 +76,7 @@ public Double visitCall(RexCall call) { case NOT_EQUALS: { selectivity = computeNotEqualitySelectivity(call); + break; } case LESS_THAN_OR_EQUAL: @@ -150,10 +150,10 @@ private Double computeDisjunctionSelectivity(RexCall call) { if (tmpSelectivity == null) { tmpSelectivity = 0.99; } - tmpCardinality = m_childCardinality * tmpSelectivity; + tmpCardinality = childCardinality * tmpSelectivity; if (tmpCardinality > 1) - tmpSelectivity = (1 - tmpCardinality / m_childCardinality); + tmpSelectivity = (1 - tmpCardinality / childCardinality); else tmpSelectivity = 1.0; @@ -194,7 +194,7 @@ private Double getMaxNDV(RexCall call) { for (RexNode op : call.getOperands()) { if (op instanceof RexInputRef) { - tmpNDV = HiveRelMdDistinctRowCount.getDistinctRowCount(m_childRel, + tmpNDV = HiveRelMdDistinctRowCount.getDistinctRowCount(this.childRel, ((RexInputRef) op).getIndex()); if (tmpNDV > maxNDV) maxNDV = tmpNDV; @@ -202,7 +202,7 @@ private Double getMaxNDV(RexCall call) { irv = new InputReferencedVisitor(); irv.apply(op); for (Integer childProjIndx : irv.inputPosReferenced) { - tmpNDV = HiveRelMdDistinctRowCount.getDistinctRowCount(m_childRel, childProjIndx); + tmpNDV = HiveRelMdDistinctRowCount.getDistinctRowCount(this.childRel, childProjIndx); if (tmpNDV > maxNDV) maxNDV = tmpNDV; } @@ -213,14 +213,13 @@ private Double getMaxNDV(RexCall call) { } private boolean isPartitionPredicate(RexNode expr, RelNode r) { - if ( r instanceof ProjectRelBase ) { + if (r instanceof ProjectRelBase) { expr = RelOptUtil.pushFilterPastProject(expr, (ProjectRelBase) r); return isPartitionPredicate(expr, ((ProjectRelBase) r).getChild()); - } else if ( r instanceof FilterRelBase ) { + } else if (r instanceof FilterRelBase) { return isPartitionPredicate(expr, ((FilterRelBase) r).getChild()); - } else if ( r instanceof HiveTableScanRel ) { - RelOptHiveTable table = (RelOptHiveTable) - ((HiveTableScanRel)r).getTable(); + } else if (r instanceof HiveTableScanRel) { + RelOptHiveTable table = (RelOptHiveTable) ((HiveTableScanRel) r).getTable(); BitSet cols = RelOptUtil.InputFinder.bits(expr); return table.containsPartitionColumnsOnly(cols); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdRowCount.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdRowCount.java index 949eb19..8c7f643 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdRowCount.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdRowCount.java @@ -28,6 +28,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveTableScanRel; import org.eigenbase.rel.FilterRelBase; import org.eigenbase.rel.JoinRelBase; import org.eigenbase.rel.JoinRelType; @@ -41,6 +42,7 @@ import org.eigenbase.rel.metadata.RelMetadataQuery; import org.eigenbase.rel.rules.SemiJoinRel; import org.eigenbase.relopt.RelOptUtil; +import org.eigenbase.relopt.hep.HepRelVertex; import org.eigenbase.rex.RexBuilder; import org.eigenbase.rex.RexCall; import org.eigenbase.rex.RexInputRef; @@ -190,7 +192,7 @@ public static PKFKRelationInfo analyzeJoinForPKFK(JoinRelBase joinRel) { RelOptUtil.classifyFilters(joinRel, joinFilters, joinRel.getJoinType(), false, !joinRel.getJoinType().generatesNullsOnRight(), !joinRel .getJoinType().generatesNullsOnLeft(), joinFilters, leftFilters, - rightFilters, joinTypeHolder, false); + rightFilters); Pair joinCols = canHandleJoin(joinRel, leftFilters, rightFilters, joinFilters); @@ -270,10 +272,11 @@ public static PKFKRelationInfo analyzeJoinForPKFK(JoinRelBase joinRel) { if (pkSide == 0) { FKSideInfo fkInfo = new FKSideInfo(rightRowCount, rightNDV); + double pkSelectivity = pkSelectivity(joinRel, true, left, leftRowCount); PKSideInfo pkInfo = new PKSideInfo(leftRowCount, leftNDV, joinRel.getJoinType().generatesNullsOnRight() ? 1.0 : - isPKSideSimpleTree ? RelMetadataQuery.getSelectivity(left, leftPred) : 1.0); + pkSelectivity); return new PKFKRelationInfo(1, fkInfo, pkInfo, ndvScalingFactor, isPKSideSimpleTree); } @@ -281,10 +284,11 @@ public static PKFKRelationInfo analyzeJoinForPKFK(JoinRelBase joinRel) { if (pkSide == 1) { FKSideInfo fkInfo = new FKSideInfo(leftRowCount, leftNDV); + double pkSelectivity = pkSelectivity(joinRel, false, right, rightRowCount); PKSideInfo pkInfo = new PKSideInfo(rightRowCount, rightNDV, joinRel.getJoinType().generatesNullsOnLeft() ? 1.0 : - isPKSideSimpleTree ? RelMetadataQuery.getSelectivity(right, rightPred) : 1.0); + pkSelectivity); return new PKFKRelationInfo(1, fkInfo, pkInfo, ndvScalingFactor, isPKSideSimpleTree); } @@ -292,6 +296,23 @@ public static PKFKRelationInfo analyzeJoinForPKFK(JoinRelBase joinRel) { return null; } + private static double pkSelectivity(JoinRelBase joinRel, boolean leftChild, + RelNode child, + double childRowCount) { + if ((leftChild && joinRel.getJoinType().generatesNullsOnRight()) || + (!leftChild && joinRel.getJoinType().generatesNullsOnLeft())) { + return 1.0; + } else { + HiveTableScanRel tScan = HiveRelMdUniqueKeys.getTableScan(child, true); + if (tScan != null) { + double tRowCount = RelMetadataQuery.getRowCount(tScan); + return childRowCount / tRowCount; + } else { + return 1.0; + } + } + } + private static boolean isKey(BitSet c, RelNode rel) { boolean isKey = false; Set keys = RelMetadataQuery.getUniqueKeys(rel); @@ -384,6 +405,10 @@ static boolean check(RelNode r, int joinKey) { @Override public void visit(RelNode node, int ordinal, RelNode parent) { + if (node instanceof HepRelVertex) { + node = ((HepRelVertex) node).getCurrentRel(); + } + if (node instanceof TableAccessRelBase) { simpleTree = true; } else if (node instanceof ProjectRelBase) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdSelectivity.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdSelectivity.java index 5d9b145..49d2ee5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdSelectivity.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdSelectivity.java @@ -37,6 +37,7 @@ import org.eigenbase.rel.metadata.RelMetadataQuery; import org.eigenbase.rex.RexNode; import org.eigenbase.rex.RexUtil; +import org.eigenbase.util.Pair; import com.google.common.collect.ImmutableMap; @@ -67,7 +68,15 @@ public Double getSelectivity(HiveJoinRel j, RexNode predicate) { private Double computeInnerJoinSelectivity(HiveJoinRel j, RexNode predicate) { double ndvCrossProduct = 1; - RexNode combinedPredicate = getCombinedPredicateForJoin(j, predicate); + Pair predInfo = + getCombinedPredicateForJoin(j, predicate); + if (!predInfo.getKey()) { + return + new FilterSelectivityEstimator(j). + estimateSelectivity(predInfo.getValue()); + } + + RexNode combinedPredicate = predInfo.getValue(); JoinPredicateInfo jpi = JoinPredicateInfo.constructJoinPredicateInfo(j, combinedPredicate); ImmutableMap.Builder colStatMapBuilder = ImmutableMap @@ -175,7 +184,14 @@ protected double exponentialBackoff(List peLst, return ndvCrossProduct; } - private RexNode getCombinedPredicateForJoin(HiveJoinRel j, RexNode additionalPredicate) { + /** + * + * @param j + * @param additionalPredicate + * @return if predicate is the join condition return (true, joinCond) + * else return (false, minusPred) + */ + private Pair getCombinedPredicateForJoin(HiveJoinRel j, RexNode additionalPredicate) { RexNode minusPred = RelMdUtil.minusPreds(j.getCluster().getRexBuilder(), additionalPredicate, j.getCondition()); @@ -184,10 +200,10 @@ private RexNode getCombinedPredicateForJoin(HiveJoinRel j, RexNode additionalPre minusList.add(j.getCondition()); minusList.add(minusPred); - return RexUtil.composeConjunction(j.getCluster().getRexBuilder(), minusList, true); + return new Pair(false, minusPred); } - return j.getCondition(); + return new Pair(true,j.getCondition()); } /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdUniqueKeys.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdUniqueKeys.java index 06ff584..3221f91 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdUniqueKeys.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdUniqueKeys.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveTableScanRel; import org.apache.hadoop.hive.ql.plan.ColStatistics; +import org.eigenbase.rel.FilterRelBase; import org.eigenbase.rel.ProjectRelBase; import org.eigenbase.rel.RelNode; import org.eigenbase.rel.metadata.BuiltInMetadata; @@ -37,6 +38,7 @@ import org.eigenbase.rel.metadata.ReflectiveRelMetadataProvider; import org.eigenbase.rel.metadata.RelMdUniqueKeys; import org.eigenbase.rel.metadata.RelMetadataProvider; +import org.eigenbase.relopt.hep.HepRelVertex; import org.eigenbase.rex.RexInputRef; import org.eigenbase.rex.RexNode; @@ -59,16 +61,15 @@ */ public Set getUniqueKeys(ProjectRelBase rel, boolean ignoreNulls) { - RelNode child = rel.getChild(); + HiveTableScanRel tScan = getTableScan(rel.getChild(), false); - if (!(child instanceof HiveTableScanRel)) { + if ( tScan == null ) { Function fn = RelMdUniqueKeys.SOURCE.apply( rel.getClass(), BuiltInMetadata.UniqueKeys.class); return ((BuiltInMetadata.UniqueKeys) fn.apply(rel)) .getUniqueKeys(ignoreNulls); } - HiveTableScanRel tScan = (HiveTableScanRel) child; Map posMap = new HashMap(); int projectPos = 0; int colStatsPos = 0; @@ -112,4 +113,26 @@ return keys; } + /* + * traverse a path of Filter, Projects to get to the TableScan. + * In case of Unique keys, stop if you reach a Project, it will be handled + * by the invocation on the Project. + * In case of getting the base rowCount of a Path, keep going past a Project. + */ + static HiveTableScanRel getTableScan(RelNode r, boolean traverseProject) { + + while (r != null && !(r instanceof HiveTableScanRel)) { + if (r instanceof HepRelVertex) { + r = ((HepRelVertex) r).getCurrentRel(); + } else if (r instanceof FilterRelBase) { + r = ((FilterRelBase) r).getChild(); + } else if (traverseProject && r instanceof ProjectRelBase) { + r = ((ProjectRelBase) r).getChild(); + } else { + r = null; + } + } + return r == null ? null : (HiveTableScanRel) r; + } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java index 98723a3..58320c7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java @@ -136,6 +136,10 @@ static ASTNode selectExpr(ASTNode expr, String alias) { } static ASTNode literal(RexLiteral literal) { + return literal(literal, false); + } + + static ASTNode literal(RexLiteral literal, boolean useTypeQualInLiteral) { Object val = null; int type = 0; SqlTypeName sqlType = literal.getType().getSqlTypeName(); @@ -147,18 +151,33 @@ static ASTNode literal(RexLiteral literal) { type = HiveParser.BigintLiteral; break; case TINYINT: - val = literal.getValue3(); + if (useTypeQualInLiteral) { + val = literal.getValue3() + "Y"; + } else { + val = literal.getValue3(); + } type = HiveParser.TinyintLiteral; break; case SMALLINT: - val = literal.getValue3(); + if (useTypeQualInLiteral) { + val = literal.getValue3() + "S"; + } else { + val = literal.getValue3(); + } type = HiveParser.SmallintLiteral; break; case INTEGER: - case BIGINT: val = literal.getValue3(); type = HiveParser.BigintLiteral; break; + case BIGINT: + if (useTypeQualInLiteral) { + val = literal.getValue3() + "L"; + } else { + val = literal.getValue3(); + } + type = HiveParser.BigintLiteral; + break; case DOUBLE: val = literal.getValue3() + "D"; type = HiveParser.Number; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java index f5a704f..a217d70 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java @@ -25,6 +25,8 @@ import net.hydromatic.optiq.util.BitSets; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.optimizer.optiq.OptiqSemanticException; import org.apache.hadoop.hive.ql.optimizer.optiq.RelOptHiveTable; @@ -63,6 +65,7 @@ import com.google.common.collect.Iterables; public class ASTConverter { + private static final Log LOG = LogFactory.getLog(ASTConverter.class); private RelNode root; private HiveAST hiveAST; @@ -149,8 +152,9 @@ private ASTNode convert() { int i = 0; for (RexNode r : select.getChildExps()) { - ASTNode selectExpr = ASTBuilder.selectExpr(r.accept(new RexVisitor(schema)), select - .getRowType().getFieldNames().get(i++)); + ASTNode selectExpr = ASTBuilder.selectExpr(r.accept( + new RexVisitor(schema, r instanceof RexLiteral)), + select.getRowType().getFieldNames().get(i++)); b.add(selectExpr); } } @@ -329,10 +333,16 @@ public void visit(RelNode node, int ordinal, RelNode parent) { static class RexVisitor extends RexVisitorImpl { private final Schema schema; + private boolean useTypeQualInLiteral; protected RexVisitor(Schema schema) { + this(schema, false); + } + + protected RexVisitor(Schema schema, boolean useTypeQualInLiteral) { super(true); this.schema = schema; + this.useTypeQualInLiteral = useTypeQualInLiteral; } @Override @@ -357,7 +367,7 @@ public ASTNode visitInputRef(RexInputRef inputRef) { @Override public ASTNode visitLiteral(RexLiteral literal) { - return ASTBuilder.literal(literal); + return ASTBuilder.literal(literal, useTypeQualInLiteral); } private ASTNode getPSpecAST(RexWindow window) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/PlanModifierForASTConv.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/PlanModifierForASTConv.java index 3d90ae7..57f030b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/PlanModifierForASTConv.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/PlanModifierForASTConv.java @@ -22,13 +22,18 @@ import java.util.List; import java.util.Set; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.optimizer.optiq.HiveOptiqUtil; import org.apache.hadoop.hive.ql.optimizer.optiq.OptiqSemanticException; import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveAggregateRel; import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveProjectRel; import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveSortRel; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.eigenbase.rel.AggregateCall; import org.eigenbase.rel.AggregateRelBase; +import org.eigenbase.rel.Aggregation; import org.eigenbase.rel.EmptyRel; import org.eigenbase.rel.FilterRelBase; import org.eigenbase.rel.JoinRelBase; @@ -40,31 +45,52 @@ import org.eigenbase.rel.SingleRel; import org.eigenbase.rel.SortRel; import org.eigenbase.rel.rules.MultiJoinRel; +import org.eigenbase.relopt.RelOptUtil; import org.eigenbase.relopt.hep.HepRelVertex; import org.eigenbase.relopt.volcano.RelSubset; import org.eigenbase.reltype.RelDataType; +import org.eigenbase.reltype.RelDataTypeFactory; import org.eigenbase.rex.RexNode; +import org.eigenbase.sql.SqlKind; import org.eigenbase.util.Pair; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; public class PlanModifierForASTConv { + private static final Log LOG = LogFactory.getLog(PlanModifierForASTConv.class); public static RelNode convertOpTree(RelNode rel, List resultSchema) throws OptiqSemanticException { RelNode newTopNode = rel; + if (LOG.isDebugEnabled()) { + LOG.debug("Original plan for PlanModifier\n " + RelOptUtil.toString(newTopNode)); + } if (!(newTopNode instanceof ProjectRelBase) && !(newTopNode instanceof SortRel)) { newTopNode = introduceDerivedTable(newTopNode); + if (LOG.isDebugEnabled()) { + LOG.debug("Plan after top-level introduceDerivedTable\n " + + RelOptUtil.toString(newTopNode)); + } } convertOpTree(newTopNode, (RelNode) null); + if (LOG.isDebugEnabled()) { + LOG.debug("Plan after nested convertOpTree\n " + RelOptUtil.toString(newTopNode)); + } Pair topSelparentPair = HiveOptiqUtil.getTopLevelSelect(newTopNode); fixTopOBSchema(newTopNode, topSelparentPair, resultSchema); + if (LOG.isDebugEnabled()) { + LOG.debug("Plan after fixTopOBSchema\n " + RelOptUtil.toString(newTopNode)); + } + topSelparentPair = HiveOptiqUtil.getTopLevelSelect(newTopNode); newTopNode = renameTopLevelSelectInResultSchema(newTopNode, topSelparentPair, resultSchema); - + if (LOG.isDebugEnabled()) { + LOG.debug("Final plan after modifier\n " + RelOptUtil.toString(newTopNode)); + } return newTopNode; } @@ -108,8 +134,14 @@ private static void convertOpTree(RelNode rel, RelNode parent) { introduceDerivedTable(((HiveSortRel) rel).getChild(), rel); } } else if (rel instanceof HiveAggregateRel) { + RelNode newParent = parent; if (!validGBParent(rel, parent)) { - introduceDerivedTable(rel, parent); + newParent = introduceDerivedTable(rel, parent); + } + // check if groupby is empty and there is no other cols in aggr + // this should only happen when newParent is constant. + if (isEmptyGrpAggr(rel)) { + replaceEmptyGroupAggr(rel, newParent); } } } @@ -125,36 +157,54 @@ private static void convertOpTree(RelNode rel, RelNode parent) { private static void fixTopOBSchema(final RelNode rootRel, Pair topSelparentPair, List resultSchema) throws OptiqSemanticException { - if (topSelparentPair.getKey() instanceof SortRel - && HiveOptiqUtil.orderRelNode(topSelparentPair.getKey())) { - HiveSortRel obRel = (HiveSortRel) topSelparentPair.getKey(); - ProjectRelBase obChild = (ProjectRelBase) topSelparentPair.getValue(); - - if (obChild.getRowType().getFieldCount() > resultSchema.size()) { - RelDataType rt = obChild.getRowType(); - Set collationInputRefs = new HashSet(RelCollationImpl.ordinals(obRel - .getCollation())); - ImmutableMap.Builder inputRefToCallMapBldr = ImmutableMap.builder(); - for (int i = resultSchema.size(); i < rt.getFieldCount(); i++) { - if (collationInputRefs.contains(i)) { - inputRefToCallMapBldr.put(i, obChild.getChildExps().get(i)); - } - } + if (!(topSelparentPair.getKey() instanceof SortRel) + || !HiveOptiqUtil.orderRelNode(topSelparentPair.getKey())) { + return; + } + HiveSortRel obRel = (HiveSortRel) topSelparentPair.getKey(); + ProjectRelBase obChild = (ProjectRelBase) topSelparentPair.getValue(); + if (obChild.getRowType().getFieldCount() <= resultSchema.size()) { + return; + } - ImmutableMap inputRefToCallMap = inputRefToCallMapBldr.build(); - if ((obChild.getRowType().getFieldCount() - inputRefToCallMap.size()) == resultSchema - .size()) { - HiveProjectRel replacementProjectRel = HiveProjectRel.create(obChild.getChild(), obChild - .getChildExps().subList(0, resultSchema.size()), obChild.getRowType().getFieldNames() - .subList(0, resultSchema.size())); - obRel.replaceInput(0, replacementProjectRel); - obRel.setInputRefToCallMap(inputRefToCallMap); - } else { - throw new OptiqSemanticException( - "Result Schema didn't match Optiq Optimized Op Tree Schema"); - } + RelDataType rt = obChild.getRowType(); + @SuppressWarnings({ "unchecked", "rawtypes" }) + Set collationInputRefs = new HashSet( + RelCollationImpl.ordinals(obRel.getCollation())); + ImmutableMap.Builder inputRefToCallMapBldr = ImmutableMap.builder(); + for (int i = resultSchema.size(); i < rt.getFieldCount(); i++) { + if (collationInputRefs.contains(i)) { + inputRefToCallMapBldr.put(i, obChild.getChildExps().get(i)); } } + ImmutableMap inputRefToCallMap = inputRefToCallMapBldr.build(); + + if ((obChild.getRowType().getFieldCount() - inputRefToCallMap.size()) != resultSchema.size()) { + LOG.error(generateInvalidSchemaMessage(obChild, resultSchema, inputRefToCallMap.size())); + throw new OptiqSemanticException("Result Schema didn't match Optimized Op Tree Schema"); + } + // This removes order-by only expressions from the projections. + HiveProjectRel replacementProjectRel = HiveProjectRel.create(obChild.getChild(), obChild + .getChildExps().subList(0, resultSchema.size()), obChild.getRowType().getFieldNames() + .subList(0, resultSchema.size())); + obRel.replaceInput(0, replacementProjectRel); + obRel.setInputRefToCallMap(inputRefToCallMap); + } + + private static String generateInvalidSchemaMessage(ProjectRelBase topLevelProj, + List resultSchema, int fieldsForOB) { + String errorDesc = "Result Schema didn't match Optiq Optimized Op Tree; schema: "; + for (FieldSchema fs : resultSchema) { + errorDesc += "[" + fs.getName() + ":" + fs.getType() + "], "; + } + errorDesc += " projection fields: "; + for (RexNode exp : topLevelProj.getChildExps()) { + errorDesc += "[" + exp.toString() + ":" + exp.getType() + "], "; + } + if (fieldsForOB != 0) { + errorDesc += fieldsForOB + " fields removed due to ORDER BY "; + } + return errorDesc.substring(0, errorDesc.length() - 2); } private static RelNode renameTopLevelSelectInResultSchema(final RelNode rootRel, @@ -167,22 +217,18 @@ private static RelNode renameTopLevelSelectInResultSchema(final RelNode rootRel, // (limit)?(OB)?(ProjectRelBase).... List rootChildExps = originalProjRel.getChildExps(); if (resultSchema.size() != rootChildExps.size()) { - // this is a bug in Hive where for queries like select key,value,value - // convertRowSchemaToResultSetSchema() only returns schema containing - // key,value - // Underlying issue is much deeper because it seems like RowResolver - // itself doesnt have - // those mappings. see limit_pushdown.q & limit_pushdown_negative.q - // Till Hive issue is fixed, disable CBO for such queries. - throw new OptiqSemanticException("Result Schema didn't match Optiq Optimized Op Tree Schema"); + // Safeguard against potential issues in CBO RowResolver construction. Disable CBO for now. + LOG.error(generateInvalidSchemaMessage(originalProjRel, resultSchema, 0)); + throw new OptiqSemanticException("Result Schema didn't match Optimized Op Tree Schema"); } List newSelAliases = new ArrayList(); String colAlias; for (int i = 0; i < rootChildExps.size(); i++) { colAlias = resultSchema.get(i).getName(); - if (colAlias.startsWith("_")) + if (colAlias.startsWith("_")) { colAlias = colAlias.substring(1); + } newSelAliases.add(colAlias); } @@ -206,7 +252,7 @@ private static RelNode introduceDerivedTable(final RelNode rel) { return select; } - private static void introduceDerivedTable(final RelNode rel, RelNode parent) { + private static RelNode introduceDerivedTable(final RelNode rel, RelNode parent) { int i = 0; int pos = -1; List childList = parent.getInputs(); @@ -226,6 +272,8 @@ private static void introduceDerivedTable(final RelNode rel, RelNode parent) { RelNode select = introduceDerivedTable(rel); parent.replaceInput(pos, select); + + return select; } private static boolean validJoinParent(RelNode joinNode, RelNode parent) { @@ -310,4 +358,40 @@ private static boolean validSetopChild(RelNode setopChild) { return validChild; } + + private static boolean isEmptyGrpAggr(RelNode gbNode) { + // Verify if both groupset and aggrfunction are empty) + AggregateRelBase aggrnode = (AggregateRelBase) gbNode; + if (aggrnode.getGroupSet().isEmpty() && aggrnode.getAggCallList().isEmpty()) { + return true; + } + return false; + } + + private static void replaceEmptyGroupAggr(final RelNode rel, RelNode parent) { + // If this function is called, the parent should only include constant + List exps = parent.getChildExps(); + for (RexNode rexNode : exps) { + if (rexNode.getKind() != SqlKind.LITERAL) { + throw new RuntimeException("We expect " + parent.toString() + + " to contain only constants. However, " + rexNode.toString() + " is " + + rexNode.getKind()); + } + } + HiveAggregateRel oldAggRel = (HiveAggregateRel) rel; + RelDataTypeFactory typeFactory = oldAggRel.getCluster().getTypeFactory(); + RelDataType longType = TypeConverter.convert(TypeInfoFactory.longTypeInfo, typeFactory); + RelDataType intType = TypeConverter.convert(TypeInfoFactory.intTypeInfo, typeFactory); + // Create the dummy aggregation. + Aggregation countFn = (Aggregation) SqlFunctionConverter.getOptiqAggFn("count", + ImmutableList.of(intType), longType); + // TODO: Using 0 might be wrong; might need to walk down to find the + // proper index of a dummy. + List argList = ImmutableList.of(0); + AggregateCall dummyCall = new AggregateCall(countFn, false, argList, longType, null); + AggregateRelBase newAggRel = oldAggRel.copy(oldAggRel.getTraitSet(), oldAggRel.getChild(), + oldAggRel.getGroupSet(), ImmutableList.of(dummyCall)); + RelNode select = introduceDerivedTable(newAggRel); + parent.replaceInput(0, select); + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java index ec85603..e6052e7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java @@ -82,35 +82,35 @@ private static final Log LOG = LogFactory.getLog(RexNodeConverter.class); private static class InputCtx { - private final RelDataType m_optiqInpDataType; - private final ImmutableMap m_hiveNameToPosMap; - private final RowResolver m_hiveRR; - private final int m_offsetInOptiqSchema; + private final RelDataType optiqInpDataType; + private final ImmutableMap hiveNameToPosMap; + private final RowResolver hiveRR; + private final int offsetInOptiqSchema; private InputCtx(RelDataType optiqInpDataType, ImmutableMap hiveNameToPosMap, RowResolver hiveRR, int offsetInOptiqSchema) { - m_optiqInpDataType = optiqInpDataType; - m_hiveNameToPosMap = hiveNameToPosMap; - m_hiveRR = hiveRR; - m_offsetInOptiqSchema = offsetInOptiqSchema; + this.optiqInpDataType = optiqInpDataType; + this.hiveNameToPosMap = hiveNameToPosMap; + this.hiveRR = hiveRR; + this.offsetInOptiqSchema = offsetInOptiqSchema; } }; - private final RelOptCluster m_cluster; - private final ImmutableList m_inputCtxs; - private final boolean m_flattenExpr; + private final RelOptCluster cluster; + private final ImmutableList inputCtxs; + private final boolean flattenExpr; public RexNodeConverter(RelOptCluster cluster, RelDataType inpDataType, ImmutableMap nameToPosMap, int offset, boolean flattenExpr) { - this.m_cluster = cluster; - m_inputCtxs = ImmutableList.of(new InputCtx(inpDataType, nameToPosMap, null, offset)); - m_flattenExpr = flattenExpr; + this.cluster = cluster; + this.inputCtxs = ImmutableList.of(new InputCtx(inpDataType, nameToPosMap, null, offset)); + this.flattenExpr = flattenExpr; } public RexNodeConverter(RelOptCluster cluster, List inpCtxLst, boolean flattenExpr) { - this.m_cluster = cluster; - m_inputCtxs = ImmutableList. builder().addAll(inpCtxLst).build(); - m_flattenExpr = flattenExpr; + this.cluster = cluster; + this.inputCtxs = ImmutableList. builder().addAll(inpCtxLst).build(); + this.flattenExpr = flattenExpr; } public RexNode convert(ExprNodeDesc expr) throws SemanticException { @@ -134,7 +134,7 @@ private RexNode convert(final ExprNodeFieldDesc fieldDesc) throws SemanticExcept RexNode rexNode = convert(fieldDesc.getDesc()); if (rexNode instanceof RexCall) { // regular case of accessing nested field in a column - return m_cluster.getRexBuilder().makeFieldAccess(rexNode, fieldDesc.getFieldName(), true); + return cluster.getRexBuilder().makeFieldAccess(rexNode, fieldDesc.getFieldName(), true); } else { // This may happen for schema-less tables, where columns are dynamically // supplied by serdes. @@ -184,7 +184,7 @@ private RexNode convert(final ExprNodeGenericFuncDesc func) throws SemanticExcep } } - argTypeBldr.add(TypeConverter.convert(tmpExprNode.getTypeInfo(), m_cluster.getTypeFactory())); + argTypeBldr.add(TypeConverter.convert(tmpExprNode.getTypeInfo(), cluster.getTypeFactory())); tmpRN = convert(tmpExprNode); childRexNodeLst.add(tmpRN); } @@ -196,20 +196,20 @@ private RexNode convert(final ExprNodeGenericFuncDesc func) throws SemanticExcep if (expr == null) { // This is not a cast; process the function. - retType = TypeConverter.convert(func.getTypeInfo(), m_cluster.getTypeFactory()); + retType = TypeConverter.convert(func.getTypeInfo(), cluster.getTypeFactory()); SqlOperator optiqOp = SqlFunctionConverter.getOptiqOperator(func.getFuncText(), func.getGenericUDF(), argTypeBldr.build(), retType); - expr = m_cluster.getRexBuilder().makeCall(optiqOp, childRexNodeLst); + expr = cluster.getRexBuilder().makeCall(optiqOp, childRexNodeLst); } else { retType = expr.getType(); } // TODO: Cast Function in Optiq have a bug where it infertype on cast throws // an exception - if (m_flattenExpr && (expr instanceof RexCall) + if (flattenExpr && (expr instanceof RexCall) && !(((RexCall) expr).getOperator() instanceof SqlCastFunction)) { RexCall call = (RexCall) expr; - expr = m_cluster.getRexBuilder().makeCall(retType, call.getOperator(), + expr = cluster.getRexBuilder().makeCall(retType, call.getOperator(), RexUtil.flatten(call.getOperands(), call.getOperator())); } @@ -246,8 +246,8 @@ private RexNode handleExplicitCast(ExprNodeGenericFuncDesc func, List c if ((udf instanceof GenericUDFToChar) || (udf instanceof GenericUDFToVarchar) || (udf instanceof GenericUDFToDecimal) || (udf instanceof GenericUDFToDate) || (udf instanceof GenericUDFToBinary) || castExprUsingUDFBridge(udf)) { - castExpr = m_cluster.getRexBuilder().makeAbstractCast( - TypeConverter.convert(func.getTypeInfo(), m_cluster.getTypeFactory()), + castExpr = cluster.getRexBuilder().makeAbstractCast( + TypeConverter.convert(func.getTypeInfo(), cluster.getTypeFactory()), childRexNodeLst.get(0)); } } @@ -258,15 +258,15 @@ private RexNode handleExplicitCast(ExprNodeGenericFuncDesc func, List c private InputCtx getInputCtx(ExprNodeColumnDesc col) throws SemanticException { InputCtx ctxLookingFor = null; - if (m_inputCtxs.size() == 1) { - ctxLookingFor = m_inputCtxs.get(0); + if (inputCtxs.size() == 1) { + ctxLookingFor = inputCtxs.get(0); } else { String tableAlias = col.getTabAlias(); String colAlias = col.getColumn(); int noInp = 0; - for (InputCtx ic : m_inputCtxs) { - if (tableAlias == null || ic.m_hiveRR.hasTableAlias(tableAlias)) { - if (ic.m_hiveRR.getPosition(colAlias) >= 0) { + for (InputCtx ic : inputCtxs) { + if (tableAlias == null || ic.hiveRR.hasTableAlias(tableAlias)) { + if (ic.hiveRR.getPosition(colAlias) >= 0) { ctxLookingFor = ic; noInp++; } @@ -282,16 +282,16 @@ private InputCtx getInputCtx(ExprNodeColumnDesc col) throws SemanticException { protected RexNode convert(ExprNodeColumnDesc col) throws SemanticException { InputCtx ic = getInputCtx(col); - int pos = ic.m_hiveNameToPosMap.get(col.getColumn()); - return m_cluster.getRexBuilder().makeInputRef( - ic.m_optiqInpDataType.getFieldList().get(pos).getType(), pos + ic.m_offsetInOptiqSchema); + int pos = ic.hiveNameToPosMap.get(col.getColumn()); + return cluster.getRexBuilder().makeInputRef( + ic.optiqInpDataType.getFieldList().get(pos).getType(), pos + ic.offsetInOptiqSchema); } private static final BigInteger MIN_LONG_BI = BigInteger.valueOf(Long.MIN_VALUE), MAX_LONG_BI = BigInteger.valueOf(Long.MAX_VALUE); protected RexNode convert(ExprNodeConstantDesc literal) throws OptiqSemanticException { - RexBuilder rexBuilder = m_cluster.getRexBuilder(); + RexBuilder rexBuilder = cluster.getRexBuilder(); RelDataTypeFactory dtFactory = rexBuilder.getTypeFactory(); PrimitiveTypeInfo hiveType = (PrimitiveTypeInfo) literal.getTypeInfo(); RelDataType optiqDataType = TypeConverter.convert(hiveType, dtFactory); @@ -314,7 +314,7 @@ protected RexNode convert(ExprNodeConstantDesc literal) throws OptiqSemanticExce optiqLiteral = rexBuilder.makeBinaryLiteral(bs); break; case SHORT: - optiqLiteral = rexBuilder.makeExactLiteral(new BigDecimal((Short) value)); + optiqLiteral = rexBuilder.makeExactLiteral(new BigDecimal((Short) value), optiqDataType); break; case INT: optiqLiteral = rexBuilder.makeExactLiteral(new BigDecimal((Integer) value)); @@ -353,7 +353,7 @@ protected RexNode convert(ExprNodeConstantDesc literal) throws OptiqSemanticExce // will work... // An alternative would be to throw CboSemanticException and fall back // to no CBO. - RelDataType relType = m_cluster.getTypeFactory().createSqlType(SqlTypeName.DECIMAL, + RelDataType relType = cluster.getTypeFactory().createSqlType(SqlTypeName.DECIMAL, bd.scale(), unscaled.toString().length()); optiqLiteral = rexBuilder.makeExactLiteral(bd, relType); } @@ -397,8 +397,8 @@ protected RexNode convert(ExprNodeConstantDesc literal) throws OptiqSemanticExce } private RexNode createNullLiteral(ExprNodeDesc expr) throws OptiqSemanticException { - return m_cluster.getRexBuilder().makeNullLiteral( - TypeConverter.convert(expr.getTypeInfo(), m_cluster.getTypeFactory()).getSqlTypeName()); + return cluster.getRexBuilder().makeNullLiteral( + TypeConverter.convert(expr.getTypeInfo(), cluster.getTypeFactory()).getSqlTypeName()); } public static RexNode convert(RelOptCluster cluster, ExprNodeDesc joinCondnExprNode, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java index 31f906a..793f117 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java @@ -64,7 +64,7 @@ import com.google.common.collect.Maps; public class SqlFunctionConverter { - private static final Log LOG = LogFactory.getLog(SqlFunctionConverter.class); + private static final Log LOG = LogFactory.getLog(SqlFunctionConverter.class); static final Map hiveToOptiq; static final Map optiqToHiveToken; @@ -87,10 +87,12 @@ public static SqlOperator getOptiqOperator(String funcTextName, GenericUDF hiveU } // do generic lookup String name = null; if (StringUtils.isEmpty(funcTextName)) { - name = getName(hiveUDF); // this should probably never happen, see getName comment + name = getName(hiveUDF); // this should probably never happen, see getName + // comment LOG.warn("The function text was empty, name from annotation is " + name); } else { - // We could just do toLowerCase here and let SA qualify it, but let's be proper... + // We could just do toLowerCase here and let SA qualify it, but let's be + // proper... name = FunctionRegistry.getNormalizedFunctionName(funcTextName); } return getOptiqFn(name, optiqArgTypes, retType); @@ -115,11 +117,10 @@ private static FunctionInfo handleExplicitCast(SqlOperator op, RelDataType dt) { if (castType.equals(TypeInfoFactory.byteTypeInfo)) { castUDF = FunctionRegistry.getFunctionInfo("tinyint"); } else if (castType instanceof CharTypeInfo) { - castUDF = handleCastForParameterizedType(castType, - FunctionRegistry.getFunctionInfo("char")); + castUDF = handleCastForParameterizedType(castType, FunctionRegistry.getFunctionInfo("char")); } else if (castType instanceof VarcharTypeInfo) { castUDF = handleCastForParameterizedType(castType, - FunctionRegistry.getFunctionInfo("varchar")); + FunctionRegistry.getFunctionInfo("varchar")); } else if (castType.equals(TypeInfoFactory.stringTypeInfo)) { castUDF = FunctionRegistry.getFunctionInfo("string"); } else if (castType.equals(TypeInfoFactory.booleanTypeInfo)) { @@ -140,24 +141,24 @@ private static FunctionInfo handleExplicitCast(SqlOperator op, RelDataType dt) { castUDF = FunctionRegistry.getFunctionInfo("datetime"); } else if (castType instanceof DecimalTypeInfo) { castUDF = handleCastForParameterizedType(castType, - FunctionRegistry.getFunctionInfo("decimal")); + FunctionRegistry.getFunctionInfo("decimal")); } else if (castType.equals(TypeInfoFactory.binaryTypeInfo)) { castUDF = FunctionRegistry.getFunctionInfo("binary"); - } else throw new IllegalStateException("Unexpected type : " + - castType.getQualifiedName()); + } else + throw new IllegalStateException("Unexpected type : " + castType.getQualifiedName()); } return castUDF; } private static FunctionInfo handleCastForParameterizedType(TypeInfo ti, FunctionInfo fi) { - SettableUDF udf = (SettableUDF)fi.getGenericUDF(); + SettableUDF udf = (SettableUDF) fi.getGenericUDF(); try { udf.setTypeInfo(ti); } catch (UDFArgumentException e) { throw new RuntimeException(e); } - return new FunctionInfo(fi.isNative(),fi.getDisplayName(),(GenericUDF)udf); + return new FunctionInfo(fi.isNative(), fi.getDisplayName(), (GenericUDF) udf); } // TODO: 1) handle Agg Func Name translation 2) is it correct to add func args @@ -175,11 +176,10 @@ public static ASTNode buildAST(SqlOperator op, List children) { } else if (op.kind == SqlKind.PLUS_PREFIX) { node = (ASTNode) ParseDriver.adaptor.create(HiveParser.PLUS, "PLUS"); } else { - if (op.getName().toUpperCase() - .equals(SqlStdOperatorTable.COUNT.getName()) + if (op.getName().toUpperCase().equals(SqlStdOperatorTable.COUNT.getName()) && children.size() == 0) { - node = (ASTNode) ParseDriver.adaptor.create( - HiveParser.TOK_FUNCTIONSTAR, "TOK_FUNCTIONSTAR"); + node = (ASTNode) ParseDriver.adaptor.create(HiveParser.TOK_FUNCTIONSTAR, + "TOK_FUNCTIONSTAR"); } node.addChild((ASTNode) ParseDriver.adaptor.create(HiveParser.Identifier, op.getName())); } @@ -210,9 +210,12 @@ public static ASTNode buildAST(SqlOperator op, List children, int i) { } - // TODO: this is not valid. Function names for built-in UDFs are specified in FunctionRegistry, - // and only happen to match annotations. For user UDFs, the name is what user specifies at - // creation time (annotation can be absent, different, or duplicate some other function). + // TODO: this is not valid. Function names for built-in UDFs are specified in + // FunctionRegistry, + // and only happen to match annotations. For user UDFs, the name is what user + // specifies at + // creation time (annotation can be absent, different, or duplicate some other + // function). private static String getName(GenericUDF hiveUDF) { String udfName = null; if (hiveUDF instanceof GenericUDFBridge) { @@ -287,70 +290,72 @@ private static HiveToken hToken(int type, String text) { } public static class OptiqUDAF extends SqlAggFunction { - final ImmutableList m_argTypes; - final RelDataType m_retType; + final ImmutableList argTypes; + final RelDataType retType; public OptiqUDAF(String opName, SqlReturnTypeInference returnTypeInference, SqlOperandTypeInference operandTypeInference, SqlOperandTypeChecker operandTypeChecker, ImmutableList argTypes, RelDataType retType) { super(opName, SqlKind.OTHER_FUNCTION, returnTypeInference, operandTypeInference, operandTypeChecker, SqlFunctionCategory.USER_DEFINED_FUNCTION); - m_argTypes = argTypes; - m_retType = retType; + this.argTypes = argTypes; + this.retType = retType; } @Override public List getParameterTypes(final RelDataTypeFactory typeFactory) { - return m_argTypes; + return this.argTypes; } @Override public RelDataType getReturnType(final RelDataTypeFactory typeFactory) { - return m_retType; + return this.retType; } } private static class OptiqUDFInfo { - private String m_udfName; - private SqlReturnTypeInference m_returnTypeInference; - private SqlOperandTypeInference m_operandTypeInference; - private SqlOperandTypeChecker m_operandTypeChecker; - private ImmutableList m_argTypes; - private RelDataType m_retType; + private String udfName; + private SqlReturnTypeInference returnTypeInference; + private SqlOperandTypeInference operandTypeInference; + private SqlOperandTypeChecker operandTypeChecker; + private ImmutableList argTypes; + private RelDataType retType; } private static OptiqUDFInfo getUDFInfo(String hiveUdfName, ImmutableList optiqArgTypes, RelDataType optiqRetType) { OptiqUDFInfo udfInfo = new OptiqUDFInfo(); - udfInfo.m_udfName = hiveUdfName; - udfInfo.m_returnTypeInference = ReturnTypes.explicit(optiqRetType); - udfInfo.m_operandTypeInference = InferTypes.explicit(optiqArgTypes); + udfInfo.udfName = hiveUdfName; + udfInfo.returnTypeInference = ReturnTypes.explicit(optiqRetType); + udfInfo.operandTypeInference = InferTypes.explicit(optiqArgTypes); ImmutableList.Builder typeFamilyBuilder = new ImmutableList.Builder(); for (RelDataType at : optiqArgTypes) { typeFamilyBuilder.add(Util.first(at.getSqlTypeName().getFamily(), SqlTypeFamily.ANY)); } - udfInfo.m_operandTypeChecker = OperandTypes.family(typeFamilyBuilder.build()); + udfInfo.operandTypeChecker = OperandTypes.family(typeFamilyBuilder.build()); - udfInfo.m_argTypes = ImmutableList. copyOf(optiqArgTypes); - udfInfo.m_retType = optiqRetType; + udfInfo.argTypes = ImmutableList. copyOf(optiqArgTypes); + udfInfo.retType = optiqRetType; return udfInfo; } public static SqlOperator getOptiqFn(String hiveUdfName, - ImmutableList optiqArgTypes, RelDataType optiqRetType) throws OptiqSemanticException{ + ImmutableList optiqArgTypes, RelDataType optiqRetType) + throws OptiqSemanticException { if (hiveUdfName != null && hiveUdfName.trim().equals("<=>")) { // We can create Optiq IS_DISTINCT_FROM operator for this. But since our - // join reordering algo cant handle this anyway there is no advantage of this. + // join reordering algo cant handle this anyway there is no advantage of + // this. // So, bail out for now. throw new OptiqSemanticException("<=> is not yet supported for cbo."); } SqlOperator optiqOp = hiveToOptiq.get(hiveUdfName); if (optiqOp == null) { OptiqUDFInfo uInf = getUDFInfo(hiveUdfName, optiqArgTypes, optiqRetType); - optiqOp = new SqlFunction(uInf.m_udfName, SqlKind.OTHER_FUNCTION, uInf.m_returnTypeInference, - uInf.m_operandTypeInference, uInf.m_operandTypeChecker, + optiqOp = new SqlFunction(uInf.udfName, SqlKind.OTHER_FUNCTION, uInf.returnTypeInference, + uInf.operandTypeInference, uInf.operandTypeChecker, SqlFunctionCategory.USER_DEFINED_FUNCTION); } @@ -363,8 +368,8 @@ public static SqlAggFunction getOptiqAggFn(String hiveUdfName, if (optiqAggFn == null) { OptiqUDFInfo uInf = getUDFInfo(hiveUdfName, optiqArgTypes, optiqRetType); - optiqAggFn = new OptiqUDAF(uInf.m_udfName, uInf.m_returnTypeInference, - uInf.m_operandTypeInference, uInf.m_operandTypeChecker, uInf.m_argTypes, uInf.m_retType); + optiqAggFn = new OptiqUDAF(uInf.udfName, uInf.returnTypeInference, uInf.operandTypeInference, + uInf.operandTypeChecker, uInf.argTypes, uInf.retType); } return optiqAggFn; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrOpProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrOpProcFactory.java index c2d49f6..8955cbd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrOpProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrOpProcFactory.java @@ -31,10 +31,13 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcFactory; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.PrunedPartitionList; import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.plan.OperatorDesc; /** @@ -133,7 +136,15 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, if (wrapper.state == PcrExprProcFactory.WalkState.TRUE) { owc.getOpToRemove().add(new PcrOpWalkerCtx.OpToDeleteInfo(pop, fop)); - } else if (wrapper.state != PcrExprProcFactory.WalkState.FALSE) { + } else if (wrapper.state == PcrExprProcFactory.WalkState.CONSTANT && wrapper.outExpr instanceof ExprNodeGenericFuncDesc) { + ExprNodeDesc desc = ConstantPropagateProcFactory.foldExpr((ExprNodeGenericFuncDesc)wrapper.outExpr); + if (desc != null && desc instanceof ExprNodeConstantDesc && Boolean.TRUE.equals(((ExprNodeConstantDesc)desc).getValue())) { + owc.getOpToRemove().add(new PcrOpWalkerCtx.OpToDeleteInfo(pop, fop)); + } else { + fop.getConf().setPredicate(wrapper.outExpr); + } + } + else if (wrapper.state != PcrExprProcFactory.WalkState.FALSE) { fop.getConf().setPredicate(wrapper.outExpr); } else { LOG.warn("Filter passes no row"); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java index 050a9a5..1d2a59f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java @@ -21,6 +21,7 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; @@ -28,6 +29,7 @@ import java.util.Map; import java.util.Set; import java.util.Stack; +import java.util.TreeMap; import java.util.regex.Pattern; import org.apache.commons.logging.Log; @@ -67,7 +69,6 @@ import org.apache.hadoop.hive.ql.plan.GroupByDesc; import org.apache.hadoop.hive.ql.plan.MapJoinDesc; import org.apache.hadoop.hive.ql.plan.MapWork; -import org.apache.hadoop.hive.ql.plan.MapredWork; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.PartitionDesc; import org.apache.hadoop.hive.ql.plan.ReduceWork; @@ -121,7 +122,11 @@ import org.apache.hadoop.hive.serde2.SerDe; import org.apache.hadoop.hive.serde2.SerDeUtils; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.util.ReflectionUtils; public class Vectorizer implements PhysicalPlanResolver { @@ -282,13 +287,13 @@ public Vectorizer() { private PhysicalContext pctx; - private int keyColCount; - private int valueColCount; + private List reduceColumnNames; + private List reduceTypeInfos; public VectorizationDispatcher(PhysicalContext pctx) { this.pctx = pctx; - keyColCount = 0; - valueColCount = 0; + reduceColumnNames = null; + reduceTypeInfos = null; } @Override @@ -385,14 +390,13 @@ private void vectorizeMapWork(MapWork mapWork) throws SemanticException { HashMap nodeOutput = new HashMap(); ogw.startWalking(topNodes, nodeOutput); - Map> columnVectorTypes = vnp.getScratchColumnVectorTypes(); - mapWork.setScratchColumnVectorTypes(columnVectorTypes); - Map> columnMap = vnp.getScratchColumnMap(); - mapWork.setScratchColumnMap(columnMap); + Map> allScratchColumnVectorTypeMaps = vnp.getAllScratchColumnVectorTypeMaps(); + mapWork.setAllScratchColumnVectorTypeMaps(allScratchColumnVectorTypeMaps); + Map> allColumnVectorMaps = vnp.getAllColumnVectorMaps(); + mapWork.setAllColumnVectorMaps(allColumnVectorMaps); if (LOG.isDebugEnabled()) { - LOG.debug(String.format("vectorTypes: %s", columnVectorTypes.toString())); - LOG.debug(String.format("columnMap: %s", columnMap.toString())); + debugDisplayAllMaps(allColumnVectorMaps, allScratchColumnVectorTypeMaps); } return; @@ -413,7 +417,7 @@ private boolean getOnlyStructObjectInspectors(ReduceWork reduceWork) throws Sema return false; } StructObjectInspector keyStructObjectInspector = (StructObjectInspector)keyObjectInspector; - keyColCount = keyStructObjectInspector.getAllStructFieldRefs().size(); + List keyFields = keyStructObjectInspector.getAllStructFieldRefs(); // Tez doesn't use tagging... if (reduceWork.getNeedsTagging()) { @@ -426,9 +430,20 @@ private boolean getOnlyStructObjectInspectors(ReduceWork reduceWork) throws Sema !(valueObjectInspector instanceof StructObjectInspector)) { return false; } - StructObjectInspector valueStructObjectInspector = - (StructObjectInspector)valueObjectInspector; - valueColCount = valueStructObjectInspector.getAllStructFieldRefs().size(); + StructObjectInspector valueStructObjectInspector = (StructObjectInspector)valueObjectInspector; + List valueFields = valueStructObjectInspector.getAllStructFieldRefs(); + + reduceColumnNames = new ArrayList(); + reduceTypeInfos = new ArrayList(); + + for (StructField field: keyFields) { + reduceColumnNames.add(Utilities.ReduceField.KEY.toString() + "." + field.getFieldName()); + reduceTypeInfos.add(TypeInfoUtils.getTypeInfoFromTypeString(field.getFieldObjectInspector().getTypeName())); + } + for (StructField field: valueFields) { + reduceColumnNames.add(Utilities.ReduceField.VALUE.toString() + "." + field.getFieldName()); + reduceTypeInfos.add(TypeInfoUtils.getTypeInfoFromTypeString(field.getFieldObjectInspector().getTypeName())); + } } catch (Exception e) { throw new SemanticException(e); } @@ -478,7 +493,7 @@ private void vectorizeReduceWork(ReduceWork reduceWork) throws SemanticException // VectorizationContext... Do we use PreOrderWalker instead of DefaultGraphWalker. Map opRules = new LinkedHashMap(); ReduceWorkVectorizationNodeProcessor vnp = - new ReduceWorkVectorizationNodeProcessor(reduceWork, keyColCount, valueColCount); + new ReduceWorkVectorizationNodeProcessor(reduceColumnNames); addReduceWorkRules(opRules, vnp); Dispatcher disp = new DefaultRuleDispatcher(vnp, opRules, null); GraphWalker ogw = new PreOrderWalker(disp); @@ -495,18 +510,17 @@ private void vectorizeReduceWork(ReduceWork reduceWork) throws SemanticException Operator reducer = reduceWork.getReducer(); if (reducer.getType().equals(OperatorType.EXTRACT)) { - ((VectorExtractOperator)reducer).setKeyAndValueColCounts(keyColCount, valueColCount); + ((VectorExtractOperator)reducer).setReduceTypeInfos(reduceTypeInfos); } - Map> columnVectorTypes = vnp.getScratchColumnVectorTypes(); - reduceWork.setScratchColumnVectorTypes(columnVectorTypes); - Map> columnMap = vnp.getScratchColumnMap(); - reduceWork.setScratchColumnMap(columnMap); + Map> allScratchColumnVectorTypeMaps = vnp.getAllScratchColumnVectorTypeMaps(); + reduceWork.setAllScratchColumnVectorTypeMaps(allScratchColumnVectorTypeMaps); + Map> allColumnVectorMaps = vnp.getAllColumnVectorMaps(); + reduceWork.setAllColumnVectorMaps(allColumnVectorMaps); if (LOG.isDebugEnabled()) { - LOG.debug(String.format("vectorTypes: %s", columnVectorTypes.toString())); - LOG.debug(String.format("columnMap: %s", columnMap.toString())); + debugDisplayAllMaps(allColumnVectorMaps, allScratchColumnVectorTypeMaps); } } } @@ -571,26 +585,26 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, protected final Set> opsDone = new HashSet>(); - public Map> getScratchColumnVectorTypes() { - Map> scratchColumnVectorTypes = + public Map> getAllScratchColumnVectorTypeMaps() { + Map> allScratchColumnVectorTypeMaps = new HashMap>(); for (String onefile : scratchColumnContext.keySet()) { VectorizationContext vc = scratchColumnContext.get(onefile); - Map cmap = vc.getOutputColumnTypeMap(); - scratchColumnVectorTypes.put(onefile, cmap); + Map cmap = vc.getScratchColumnTypeMap(); + allScratchColumnVectorTypeMaps.put(onefile, cmap); } - return scratchColumnVectorTypes; + return allScratchColumnVectorTypeMaps; } - public Map> getScratchColumnMap() { - Map> scratchColumnMap = + public Map> getAllColumnVectorMaps() { + Map> allColumnVectorMaps = new HashMap>(); for(String oneFile: scratchColumnContext.keySet()) { VectorizationContext vc = scratchColumnContext.get(oneFile); - Map cmap = vc.getColumnMap(); - scratchColumnMap.put(oneFile, cmap); + Map cmap = vc.getProjectionColumnMap(); + allColumnVectorMaps.put(oneFile, cmap); } - return scratchColumnMap; + return allColumnVectorMaps; } public VectorizationContext walkStackToFindVectorizationContext(Stack stack, @@ -676,10 +690,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, vContext.setFileKey(onefile); scratchColumnContext.put(onefile, vContext); if (LOG.isDebugEnabled()) { - LOG.debug("Vectorized MapWork operator " + op.getName() + - " with vectorization context key=" + vContext.getFileKey() + - ", vectorTypes: " + vContext.getOutputColumnTypeMap().toString() + - ", columnMap: " + vContext.getColumnMap().toString()); + LOG.debug("Vectorized MapWork operator " + op.getName() + " vectorization context " + vContext.toString()); } break; } @@ -710,17 +721,11 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Operator vectorOp = doVectorize(op, vContext); if (LOG.isDebugEnabled()) { - LOG.debug("Vectorized MapWork operator " + vectorOp.getName() + - " with vectorization context key=" + vContext.getFileKey() + - ", vectorTypes: " + vContext.getOutputColumnTypeMap().toString() + - ", columnMap: " + vContext.getColumnMap().toString()); + LOG.debug("Vectorized MapWork operator " + vectorOp.getName() + " vectorization context " + vContext.toString()); if (vectorOp instanceof VectorizationContextRegion) { VectorizationContextRegion vcRegion = (VectorizationContextRegion) vectorOp; VectorizationContext vOutContext = vcRegion.getOuputVectorizationContext(); - LOG.debug("Vectorized MapWork operator " + vectorOp.getName() + - " added new vectorization context key=" + vOutContext.getFileKey() + - ", vectorTypes: " + vOutContext.getOutputColumnTypeMap().toString() + - ", columnMap: " + vOutContext.getColumnMap().toString()); + LOG.debug("Vectorized MapWork operator " + vectorOp.getName() + " added vectorization context " + vContext.toString()); } } @@ -730,10 +735,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, class ReduceWorkVectorizationNodeProcessor extends VectorizationNodeProcessor { - private final ReduceWork rWork; - private int keyColCount; - private int valueColCount; - private Map reduceColumnNameMap; + private List reduceColumnNames; private VectorizationContext reduceShuffleVectorizationContext; @@ -743,12 +745,8 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, return rootVectorOp; } - public ReduceWorkVectorizationNodeProcessor(ReduceWork rWork, int keyColCount, - int valueColCount) { - this.rWork = rWork; - reduceColumnNameMap = rWork.getReduceColumnNameMap(); - this.keyColCount = keyColCount; - this.valueColCount = valueColCount; + public ReduceWorkVectorizationNodeProcessor(List reduceColumnNames) { + this.reduceColumnNames = reduceColumnNames; rootVectorOp = null; reduceShuffleVectorizationContext = null; } @@ -766,17 +764,16 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, boolean saveRootVectorOp = false; if (op.getParentOperators().size() == 0) { - vContext = getReduceVectorizationContext(reduceColumnNameMap); + LOG.info("ReduceWorkVectorizationNodeProcessor process reduceColumnNames " + reduceColumnNames.toString()); + + vContext = new VectorizationContext(reduceColumnNames); vContext.setFileKey("_REDUCE_SHUFFLE_"); scratchColumnContext.put("_REDUCE_SHUFFLE_", vContext); reduceShuffleVectorizationContext = vContext; saveRootVectorOp = true; if (LOG.isDebugEnabled()) { - LOG.debug("Vectorized ReduceWork reduce shuffle vectorization context key=" + - vContext.getFileKey() + - ", vectorTypes: " + vContext.getOutputColumnTypeMap().toString() + - ", columnMap: " + vContext.getColumnMap().toString()); + LOG.debug("Vectorized ReduceWork reduce shuffle vectorization context " + vContext.toString()); } } else { vContext = walkStackToFindVectorizationContext(stack, op); @@ -802,17 +799,11 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Operator vectorOp = doVectorize(op, vContext); if (LOG.isDebugEnabled()) { - LOG.debug("Vectorized ReduceWork operator " + vectorOp.getName() + - " with vectorization context key=" + vContext.getFileKey() + - ", vectorTypes: " + vContext.getOutputColumnTypeMap().toString() + - ", columnMap: " + vContext.getColumnMap().toString()); + LOG.debug("Vectorized ReduceWork operator " + vectorOp.getName() + " vectorization context " + vContext.toString()); if (vectorOp instanceof VectorizationContextRegion) { VectorizationContextRegion vcRegion = (VectorizationContextRegion) vectorOp; VectorizationContext vOutContext = vcRegion.getOuputVectorizationContext(); - LOG.debug("Vectorized ReduceWork operator " + vectorOp.getName() + - " added new vectorization context key=" + vOutContext.getFileKey() + - ", vectorTypes: " + vOutContext.getOutputColumnTypeMap().toString() + - ", columnMap: " + vOutContext.getColumnMap().toString()); + LOG.debug("Vectorized ReduceWork operator " + vectorOp.getName() + " added vectorization context " + vContext.toString()); } } if (vectorOp instanceof VectorGroupByOperator) { @@ -830,7 +821,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, private static class ValidatorVectorizationContext extends VectorizationContext { private ValidatorVectorizationContext() { - super(null, -1); + super(); } @Override @@ -918,7 +909,12 @@ boolean validateReduceWorkOperator(Operator op) { } break; case GROUPBY: - ret = validateGroupByOperator((GroupByOperator) op, true, true); + if (HiveConf.getBoolVar(physicalContext.getConf(), + HiveConf.ConfVars.HIVE_VECTORIZATION_REDUCE_GROUPBY_ENABLED)) { + ret = validateGroupByOperator((GroupByOperator) op, true, true); + } else { + ret = false; + } break; case FILTER: ret = validateFilterOperator((FilterOperator) op); @@ -1026,12 +1022,10 @@ private boolean validateGroupByOperator(GroupByOperator op, boolean isReduce, bo if (!ret) { return false; } - boolean isVectorOutput = isTez && aggregatorsOutputIsPrimitive(desc.getAggregators(), isReduce); - vectorDesc.setVectorOutput(isVectorOutput); if (isReduce) { if (desc.isDistinct()) { LOG.info("Distinct not supported in reduce vector mode"); - return false; + return false; } // Sort-based GroupBy? if (desc.getMode() != GroupByDesc.Mode.COMPLETE && @@ -1044,21 +1038,24 @@ private boolean validateGroupByOperator(GroupByOperator op, boolean isReduce, bo LOG.info("Reduce GROUP BY mode is " + desc.getMode().name()); if (desc.getGroupKeyNotReductionKey()) { LOG.info("Reduce vector mode not supported when group key is not reduction key"); - return false; + return false; } - if (!isVectorOutput) { + if (!aggregatorsOutputIsPrimitive(desc.getAggregators(), isReduce)) { LOG.info("Reduce vector mode only supported when aggregate outputs are primitive types"); - return false; + return false; } if (desc.getKeys().size() > 0) { + if (op.getParentOperators().size() > 0) { + LOG.info("Reduce vector mode can only handle a key group GROUP BY operator when it is fed by reduce-shuffle"); + return false; + } LOG.info("Reduce-side GROUP BY will process key groups"); vectorDesc.setVectorGroupBatches(true); } else { LOG.info("Reduce-side GROUP BY will do global aggregation"); } + vectorDesc.setVectorOutput(true); vectorDesc.setIsReduce(true); - } else { - LOG.info("Downstream operators of map-side GROUP BY will be vectorized: " + isVectorOutput); } return true; } @@ -1227,21 +1224,17 @@ private VectorizationContext getVectorizationContext(Operator op, PhysicalContext pctx) { RowSchema rs = op.getSchema(); - Map cmap = new HashMap(); - int columnCount = 0; + // Add all non-virtual columns to make a vectorization context for + // the TableScan operator. + VectorizationContext vContext = new VectorizationContext(); for (ColumnInfo c : rs.getSignature()) { // Earlier, validation code should have eliminated virtual columns usage (HIVE-5560). if (!isVirtualColumn(c)) { - cmap.put(c.getInternalName(), columnCount++); + vContext.addInitialColumn(c.getInternalName()); } } - - return new VectorizationContext(cmap, columnCount); - } - - private VectorizationContext getReduceVectorizationContext( - Map reduceColumnNameMap) { - return new VectorizationContext(reduceColumnNameMap, reduceColumnNameMap.size()); + vContext.finishedAddingInitialColumns(); + return vContext; } private void fixupParentChildOperators(Operator op, @@ -1297,4 +1290,41 @@ private boolean isVirtualColumn(ColumnInfo column) { } return false; } + + public void debugDisplayAllMaps(Map> allColumnVectorMaps, + Map> allScratchColumnVectorTypeMaps) { + + // Context keys grow in length since they are a path... + Comparator comparerShorterString = new Comparator() { + @Override + public int compare(String o1, String o2) { + Integer length1 = o1.length(); + Integer length2 = o2.length(); + return length1.compareTo(length2); + }}; + + Comparator comparerInteger = new Comparator() { + @Override + public int compare(Integer o1, Integer o2) { + return o1.compareTo(o2); + }}; + + Map> sortedAllColumnVectorMaps = new TreeMap>(comparerShorterString); + for (Map.Entry> entry : allColumnVectorMaps.entrySet()) { + Map sortedColumnMap = new TreeMap(comparerInteger); + for (Map.Entry innerEntry : entry.getValue().entrySet()) { + sortedColumnMap.put(innerEntry.getValue(), innerEntry.getKey()); + } + sortedAllColumnVectorMaps.put(entry.getKey(), sortedColumnMap); + } + LOG.debug("sortedAllColumnVectorMaps " + sortedAllColumnVectorMaps); + + Map> sortedAllScratchColumnVectorTypeMap = new TreeMap>(comparerShorterString); + for (Map.Entry> entry : allScratchColumnVectorTypeMaps.entrySet()) { + Map sortedScratchColumnTypeMap = new TreeMap(comparerInteger); + sortedScratchColumnTypeMap.putAll(entry.getValue()); + sortedAllScratchColumnVectorTypeMap.put(entry.getKey(), sortedScratchColumnTypeMap); + } + LOG.debug("sortedAllScratchColumnVectorTypeMap " + sortedAllScratchColumnVectorTypeMap); + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java index e43d39f..1796b7b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java @@ -56,7 +56,9 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr; +import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; /** @@ -188,12 +190,18 @@ public static PrunedPartitionList prune(Table tab, ExprNodeDesc prunerExpr, // Replace virtual columns with nulls. See javadoc for details. prunerExpr = removeNonPartCols(prunerExpr, extractPartColNames(tab), partColsUsedInFilter); // Remove all parts that are not partition columns. See javadoc for details. - ExprNodeGenericFuncDesc compactExpr = (ExprNodeGenericFuncDesc)compactExpr(prunerExpr.clone()); + ExprNodeDesc compactExpr = compactExpr(prunerExpr.clone()); String oldFilter = prunerExpr.getExprString(); - if (compactExpr == null) { - // Non-strict mode, and all the predicates are on non-partition columns - get everything. - LOG.debug("Filter " + oldFilter + " was null after compacting"); - return getAllPartsFromCacheOrServer(tab, key, true, prunedPartitionsMap); + if (isBooleanExpr(compactExpr)) { + // For null and true values, return every partition + if (!isFalseExpr(compactExpr)) { + // Non-strict mode, and all the predicates are on non-partition columns - get everything. + LOG.debug("Filter " + oldFilter + " was null after compacting"); + return getAllPartsFromCacheOrServer(tab, key, true, prunedPartitionsMap); + } else { + return new PrunedPartitionList(tab, new LinkedHashSet(new ArrayList()), + new ArrayList(), false); + } } LOG.debug("Filter w/ compacting: " + compactExpr.getExprString() + "; filter w/o compacting: " + oldFilter); @@ -204,7 +212,7 @@ public static PrunedPartitionList prune(Table tab, ExprNodeDesc prunerExpr, return ppList; } - ppList = getPartitionsFromServer(tab, compactExpr, conf, alias, partColsUsedInFilter, oldFilter.equals(compactExpr.getExprString())); + ppList = getPartitionsFromServer(tab, (ExprNodeGenericFuncDesc)compactExpr, conf, alias, partColsUsedInFilter, oldFilter.equals(compactExpr.getExprString())); prunedPartitionsMap.put(key, ppList); return ppList; } @@ -225,16 +233,22 @@ private static PrunedPartitionList getAllPartsFromCacheOrServer(Table tab, Strin partsCache.put(key, ppList); return ppList; } - - private static ExprNodeDesc removeTruePredciates(ExprNodeDesc e) { - if (e instanceof ExprNodeConstantDesc) { - ExprNodeConstantDesc eC = (ExprNodeConstantDesc) e; - if (e.getTypeInfo() == TypeInfoFactory.booleanTypeInfo - && eC.getValue() == Boolean.TRUE) { - return null; - } - } - return e; + + static private boolean isBooleanExpr(ExprNodeDesc expr) { + return expr != null && expr instanceof ExprNodeConstantDesc && + ((ExprNodeConstantDesc)expr).getTypeInfo() instanceof PrimitiveTypeInfo && + ((PrimitiveTypeInfo)(((ExprNodeConstantDesc)expr).getTypeInfo())). + getTypeName().equals(serdeConstants.BOOLEAN_TYPE_NAME); + } + static private boolean isTrueExpr(ExprNodeDesc expr) { + return isBooleanExpr(expr) && + ((ExprNodeConstantDesc)expr).getValue() != null && + ((ExprNodeConstantDesc)expr).getValue().equals(Boolean.TRUE); + } + static private boolean isFalseExpr(ExprNodeDesc expr) { + return isBooleanExpr(expr) && + ((ExprNodeConstantDesc)expr).getValue() != null && + ((ExprNodeConstantDesc)expr).getValue().equals(Boolean.FALSE); } /** @@ -245,10 +259,13 @@ private static ExprNodeDesc removeTruePredciates(ExprNodeDesc e) { * @return partition pruning expression that only contains partition columns. */ static private ExprNodeDesc compactExpr(ExprNodeDesc expr) { - if (expr instanceof ExprNodeConstantDesc) { - expr = removeTruePredciates(expr); - if (expr == null || ((ExprNodeConstantDesc)expr).getValue() == null) { - return null; + // If this is a constant boolean expression, return the value. + if (expr == null) { + return null; + } + if (expr instanceof ExprNodeConstantDesc) { + if (isBooleanExpr(expr)) { + return expr; } else { throw new IllegalStateException("Unexpected non-null ExprNodeConstantDesc: " + expr.getExprString()); @@ -256,22 +273,29 @@ static private ExprNodeDesc compactExpr(ExprNodeDesc expr) { } else if (expr instanceof ExprNodeGenericFuncDesc) { GenericUDF udf = ((ExprNodeGenericFuncDesc)expr).getGenericUDF(); boolean isAnd = udf instanceof GenericUDFOPAnd; - if (isAnd || udf instanceof GenericUDFOPOr) { + boolean isOr = udf instanceof GenericUDFOPOr; + + if (isAnd || isOr) { List children = expr.getChildren(); - ExprNodeDesc left = removeTruePredciates(children.get(0)); - children.set(0, left == null ? null : compactExpr(left)); - ExprNodeDesc right = removeTruePredciates(children.get(1)); - children.set(1, right == null ? null : compactExpr(right)); - - // Note that one does not simply compact (not-null or null) to not-null. - // Only if we have an "and" is it valid to send one side to metastore. - if (children.get(0) == null && children.get(1) == null) { - return null; - } else if (children.get(0) == null) { - return isAnd ? children.get(1) : null; - } else if (children.get(1) == null) { - return isAnd ? children.get(0) : null; - } + ExprNodeDesc left = children.get(0); + children.set(0, compactExpr(left)); + ExprNodeDesc right = children.get(1); + children.set(1, compactExpr(right)); + + if (isTrueExpr(children.get(0)) && isTrueExpr(children.get(1))) { + return new ExprNodeConstantDesc(Boolean.TRUE); + } else if (isTrueExpr(children.get(0))) { + return isAnd ? children.get(1) : new ExprNodeConstantDesc(Boolean.TRUE); + } else if (isTrueExpr(children.get(1))) { + return isAnd ? children.get(0) : new ExprNodeConstantDesc(Boolean.TRUE); + } else if (isFalseExpr(children.get(0)) && isFalseExpr(children.get(1))) { + return new ExprNodeConstantDesc(Boolean.FALSE); + } else if (isFalseExpr(children.get(0))) { + return isAnd ? new ExprNodeConstantDesc(Boolean.FALSE) : children.get(1); + } else if (isFalseExpr(children.get(1))) { + return isAnd ? new ExprNodeConstantDesc(Boolean.FALSE) : children.get(0); + } + } return expr; } else { @@ -296,9 +320,9 @@ static private ExprNodeDesc removeNonPartCols(ExprNodeDesc expr, List pa if (!partCols.contains(column)) { // Column doesn't appear to be a partition column for the table. return new ExprNodeConstantDesc(expr.getTypeInfo(), null); - } + } referred.add(column); - } + } if (expr instanceof ExprNodeGenericFuncDesc) { List children = expr.getChildren(); for (int i = 0; i < children.size(); ++i) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SetSparkReducerParallelism.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SetSparkReducerParallelism.java index f3bdeee..2dbb5a3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SetSparkReducerParallelism.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SetSparkReducerParallelism.java @@ -85,7 +85,6 @@ public Object process(Node nd, Stack stack, maxReducers, false); LOG.info("Set parallelism for reduce sink " + sink + " to: " + numReducers); desc.setNumReducers(numReducers); - desc.setAutoParallel(true); } } else { LOG.info("Number of reducers determined to be: " + desc.getNumReducers()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java index ab7eee8..d49eddb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java @@ -42,7 +42,6 @@ import org.apache.hadoop.hive.ql.exec.RowSchema; import org.apache.hadoop.hive.ql.exec.SelectOperator; import org.apache.hadoop.hive.ql.exec.TableScanOperator; -import org.apache.hadoop.hive.ql.exec.tez.DagUtils; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; @@ -151,28 +150,28 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Statistics parentStats = parent.getStatistics(); AnnotateStatsProcCtx aspCtx = (AnnotateStatsProcCtx) procCtx; HiveConf conf = aspCtx.getConf(); + Statistics stats = null; - // SELECT (*) does not change the statistics. Just pass on the parent statistics - if (sop.getConf().isSelectStar()) { + if (parentStats != null) { try { - if (parentStats != null) { - sop.setStatistics(parentStats.clone()); - } + stats = parentStats.clone(); } catch (CloneNotSupportedException e) { throw new SemanticException(ErrorMsg.STATISTICS_CLONING_FAILED.getMsg()); } - return null; } try { if (satisfyPrecondition(parentStats)) { - Statistics stats = parentStats.clone(); - List colStats = - StatsUtils.getColStatisticsFromExprMap(conf, parentStats, sop.getColumnExprMap(), - sop.getSchema()); - long dataSize = StatsUtils.getDataSizeFromColumnStats(stats.getNumRows(), colStats); + // this will take care of mapping between input column names and output column names. The + // returned column stats will have the output column names. + List colStats = StatsUtils.getColStatisticsFromExprMap(conf, parentStats, + sop.getColumnExprMap(), sop.getSchema()); stats.setColumnStats(colStats); - stats.setDataSize(setMaxIfInvalid(dataSize)); + // in case of select(*) the data size does not change + if (!sop.getConf().isSelectStar() && !sop.getConf().isSelStarNoCompute()) { + long dataSize = StatsUtils.getDataSizeFromColumnStats(stats.getNumRows(), colStats); + stats.setDataSize(setMaxIfInvalid(dataSize)); + } sop.setStatistics(stats); if (isDebugEnabled) { @@ -889,16 +888,12 @@ private boolean checkMapSideAggregation(GroupByOperator gop, GroupByDesc.Mode mode = desc.getMode(); if (mode.equals(GroupByDesc.Mode.HASH)) { - float hashAggMem = conf.getFloatVar( - HiveConf.ConfVars.HIVEMAPAGGRHASHMEMORY); - float hashAggMaxThreshold = conf.getFloatVar( - HiveConf.ConfVars.HIVEMAPAGGRMEMORYTHRESHOLD); + float hashAggMem = conf.getFloatVar(HiveConf.ConfVars.HIVEMAPAGGRHASHMEMORY); + float hashAggMaxThreshold = conf.getFloatVar(HiveConf.ConfVars.HIVEMAPAGGRMEMORYTHRESHOLD); - // get memory for container. May be use mapreduce.map.java.opts instead? - long totalMemory = - DagUtils.getContainerResource(conf).getMemory() * 1000L * 1000L; - long maxMemHashAgg = Math - .round(totalMemory * hashAggMem * hashAggMaxThreshold); + // get available map memory + long totalMemory = StatsUtils.getAvailableMemory(conf) * 1000L * 1000L; + long maxMemHashAgg = Math.round(totalMemory * hashAggMem * hashAggMaxThreshold); // estimated number of rows will be product of NDVs long numEstimatedRows = 1; @@ -1026,11 +1021,17 @@ private long applyGBYRule(long numRows, long dvProd) { */ public static class JoinStatsRule extends DefaultStatsRule implements NodeProcessor { + private boolean pkfkInferred = false; + private long newNumRows = 0; + private List> parents; + private CommonJoinOperator jop; + private int numAttr = 1; + @Override public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { - CommonJoinOperator jop = (CommonJoinOperator) nd; - List> parents = jop.getParentOperators(); + jop = (CommonJoinOperator) nd; + parents = jop.getParentOperators(); AnnotateStatsProcCtx aspCtx = (AnnotateStatsProcCtx) procCtx; HiveConf conf = aspCtx.getConf(); boolean allStatsAvail = true; @@ -1057,22 +1058,26 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Statistics stats = new Statistics(); Map rowCountParents = new HashMap(); List distinctVals = Lists.newArrayList(); - - // 2 relations, multiple attributes - boolean multiAttr = false; - int numAttr = 1; int numParent = parents.size(); - Map joinedColStats = Maps.newHashMap(); Map> joinKeys = Maps.newHashMap(); List rowCounts = Lists.newArrayList(); + // detect if there are multiple attributes in join key + ReduceSinkOperator rsOp = (ReduceSinkOperator) jop.getParentOperators().get(0); + List keyExprs = rsOp.getConf().getKeyCols(); + numAttr = keyExprs.size(); + + // infer PK-FK relationship in single attribute join case + pkfkInferred = false; + inferPKFKRelationship(); + // get the join keys from parent ReduceSink operators for (int pos = 0; pos < parents.size(); pos++) { ReduceSinkOperator parent = (ReduceSinkOperator) jop.getParentOperators().get(pos); Statistics parentStats = parent.getStatistics(); - List keyExprs = parent.getConf().getKeyCols(); + keyExprs = parent.getConf().getKeyCols(); // Parent RS may have column statistics from multiple parents. // Populate table alias to row count map, this will be used later to @@ -1087,12 +1092,6 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, } rowCounts.add(parentStats.getNumRows()); - // multi-attribute join key - if (keyExprs.size() > 1) { - multiAttr = true; - numAttr = keyExprs.size(); - } - // compute fully qualified join key column names. this name will be // used to quickly look-up for column statistics of join key. // TODO: expressions in join condition will be ignored. assign @@ -1115,7 +1114,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, // attribute join, else max(V(R,y1), V(S,y1)) * max(V(R,y2), V(S,y2)) // in case of multi-attribute join long denom = 1; - if (multiAttr) { + if (numAttr > 1) { List perAttrDVs = Lists.newArrayList(); for (int idx = 0; idx < numAttr; idx++) { for (Integer i : joinKeys.keySet()) { @@ -1154,9 +1153,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, } // Update NDV of joined columns to be min(V(R,y), V(S,y)) - if (multiAttr) { - updateJoinColumnsNDV(joinKeys, joinedColStats, numAttr); - } + updateJoinColumnsNDV(joinKeys, joinedColStats, numAttr); // column statistics from different sources are put together and rename // fully qualified column names based on output schema of join operator @@ -1186,10 +1183,8 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, // update join statistics stats.setColumnStats(outColStats); - long newRowCount = computeNewRowCount(rowCounts, denom); - - updateStatsForJoinType(stats, newRowCount, jop, rowCountParents, - outInTabAlias); + long newRowCount = pkfkInferred ? newNumRows : computeNewRowCount(rowCounts, denom); + updateStatsForJoinType(stats, newRowCount, jop, rowCountParents,outInTabAlias); jop.setStatistics(stats); if (isDebugEnabled) { @@ -1234,6 +1229,195 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, return null; } + private void inferPKFKRelationship() { + if (numAttr == 1) { + List parentsWithPK = getPrimaryKeyCandidates(parents); + + // in case of fact to many dimensional tables join, the join key in fact table will be + // mostly foreign key which will have corresponding primary key in dimension table. + // The selectivity of fact table in that case will be product of all selectivities of + // dimension tables (assumes conjunctivity) + for (Integer id : parentsWithPK) { + ColStatistics csPK = null; + Operator parent = parents.get(id); + for (ColStatistics cs : parent.getStatistics().getColumnStats()) { + if (cs.isPrimaryKey()) { + csPK = cs; + break; + } + } + + // infer foreign key candidates positions + List parentsWithFK = getForeignKeyCandidates(parents, csPK); + if (parentsWithFK.size() == 1 && + parentsWithFK.size() + parentsWithPK.size() == parents.size()) { + Operator parentWithFK = parents.get(parentsWithFK.get(0)); + List parentsSel = getSelectivity(parents, parentsWithPK); + Float prodSelectivity = 1.0f; + for (Float selectivity : parentsSel) { + prodSelectivity *= selectivity; + } + newNumRows = (long) Math.ceil( + parentWithFK.getStatistics().getNumRows() * prodSelectivity); + pkfkInferred = true; + + // some debug information + if (isDebugEnabled) { + List parentIds = Lists.newArrayList(); + + // print primary key containing parents + for (Integer i : parentsWithPK) { + parentIds.add(parents.get(i).toString()); + } + LOG.debug("STATS-" + jop.toString() + ": PK parent id(s) - " + parentIds); + parentIds.clear(); + + // print foreign key containing parents + for (Integer i : parentsWithFK) { + parentIds.add(parents.get(i).toString()); + } + LOG.debug("STATS-" + jop.toString() + ": FK parent id(s) - " + parentIds); + } + } + } + } + } + + /** + * Get selectivity of reduce sink operators. + * @param ops - reduce sink operators + * @param opsWithPK - reduce sink operators with primary keys + * @return - list of selectivity for primary key containing operators + */ + private List getSelectivity(List> ops, + List opsWithPK) { + List result = Lists.newArrayList(); + for (Integer idx : opsWithPK) { + Operator op = ops.get(idx); + float selectivity = getSelectivitySimpleTree(op); + result.add(selectivity); + } + return result; + } + + private float getSelectivitySimpleTree(Operator op) { + TableScanOperator tsOp = OperatorUtils + .findSingleOperatorUpstream(op, TableScanOperator.class); + if (tsOp == null) { + // complex tree with multiple parents + return getSelectivityComplexTree(op); + } else { + // simple tree with single parent + long inputRow = tsOp.getStatistics().getNumRows(); + long outputRow = op.getStatistics().getNumRows(); + return (float) outputRow / (float) inputRow; + } + } + + private float getSelectivityComplexTree(Operator op) { + Operator multiParentOp = null; + Operator currentOp = op; + + // TS-1 TS-2 + // | | + // RS-1 RS-2 + // \ / + // JOIN + // | + // FIL + // | + // RS-3 + // + // For the above complex operator tree, + // selectivity(JOIN) = selectivity(RS-1) * selectivity(RS-2) and + // selectivity(RS-3) = numRows(RS-3)/numRows(JOIN) * selectivity(JOIN) + while(multiParentOp == null) { + if (op.getParentOperators().size() > 1) { + multiParentOp = op; + } else { + op = op.getParentOperators().get(0); + } + } + + float selMultiParent = 1.0f; + for(Operator parent : multiParentOp.getParentOperators()) { + // In the above example, TS-1 -> RS-1 and TS-2 -> RS-2 are simple trees + selMultiParent *= getSelectivitySimpleTree(parent); + } + + float selCurrOp = ((float) currentOp.getStatistics().getNumRows() / + (float) multiParentOp.getStatistics().getNumRows()) * selMultiParent; + + return selCurrOp; + } + + /** + * Returns the index of parents whose join key column statistics ranges are within the specified + * primary key range (inferred as foreign keys). + * @param ops - operators + * @param csPK - column statistics of primary key + * @return - list of foreign key containing parent ids + */ + private List getForeignKeyCandidates(List> ops, + ColStatistics csPK) { + List result = Lists.newArrayList(); + if (csPK == null || ops == null) { + return result; + } + + for (int i = 0; i < ops.size(); i++) { + Operator op = ops.get(i); + if (op != null && op instanceof ReduceSinkOperator) { + ReduceSinkOperator rsOp = (ReduceSinkOperator) op; + List keys = rsOp.getConf().getKeyCols(); + List fqCols = StatsUtils.getFullQualifedColNameFromExprs(keys, + rsOp.getColumnExprMap()); + if (fqCols.size() == 1) { + String joinCol = fqCols.get(0); + if (rsOp.getStatistics() != null) { + ColStatistics cs = rsOp.getStatistics().getColumnStatisticsFromFQColName(joinCol); + if (cs != null && !cs.isPrimaryKey()) { + if (StatsUtils.inferForeignKey(csPK, cs)) { + result.add(i); + } + } + } + } + } + } + return result; + } + + /** + * Returns the index of parents whose join key columns are infer as primary keys + * @param ops - operators + * @return - list of primary key containing parent ids + */ + private List getPrimaryKeyCandidates(List> ops) { + List result = Lists.newArrayList(); + if (ops != null && !ops.isEmpty()) { + for (int i = 0; i < ops.size(); i++) { + Operator op = ops.get(i); + if (op instanceof ReduceSinkOperator) { + ReduceSinkOperator rsOp = (ReduceSinkOperator) op; + List keys = rsOp.getConf().getKeyCols(); + List fqCols = StatsUtils.getFullQualifedColNameFromExprs(keys, + rsOp.getColumnExprMap()); + if (fqCols.size() == 1) { + String joinCol = fqCols.get(0); + if (rsOp.getStatistics() != null) { + ColStatistics cs = rsOp.getStatistics().getColumnStatisticsFromFQColName(joinCol); + if (cs != null && cs.isPrimaryKey()) { + result.add(i); + } + } + } + } + } + } + return result; + } + private Long getEasedOutDenominator(List distinctVals) { // Exponential back-off for NDVs. // 1) Descending order sort of NDVs @@ -1253,7 +1437,7 @@ private void updateStatsForJoinType(Statistics stats, long newNumRows, Map rowCountParents, Map outInTabAlias) { - if (newNumRows <= 0) { + if (newNumRows < 0) { LOG.info("STATS-" + jop.toString() + ": Overflow in number of rows." + newNumRows + " rows will be set to Long.MAX_VALUE"); } @@ -1528,6 +1712,8 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { Operator op = (Operator) nd; OperatorDesc conf = op.getConf(); + AnnotateStatsProcCtx aspCtx = (AnnotateStatsProcCtx) procCtx; + HiveConf hconf = aspCtx.getConf(); if (conf != null) { Statistics stats = conf.getStatistics(); @@ -1544,7 +1730,9 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, stats.addToNumRows(parentStats.getNumRows()); stats.addToDataSize(parentStats.getDataSize()); stats.updateColumnStatsState(parentStats.getColumnStatsState()); - stats.addToColumnStats(parentStats.getColumnStats()); + List colStats = StatsUtils.getColStatisticsFromExprMap(hconf, + parentStats, op.getColumnExprMap(), op.getSchema()); + stats.addToColumnStats(colStats); op.getConf().setStatistics(stats); if (isDebugEnabled) { @@ -1622,7 +1810,7 @@ static void updateStats(Statistics stats, long newNumRows, boolean useColStats, Operator op, boolean updateNDV) { - if (newNumRows <= 0) { + if (newNumRows < 0) { LOG.info("STATS-" + op.toString() + ": Overflow in number of rows." + newNumRows + " rows will be set to Long.MAX_VALUE"); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index 37cbf7f..becbc38 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -79,7 +79,7 @@ * */ public abstract class BaseSemanticAnalyzer { - private static final Log STATIC_LOG = LogFactory.getLog(BaseSemanticAnalyzer.class.getName()); + protected static final Log STATIC_LOG = LogFactory.getLog(BaseSemanticAnalyzer.class.getName()); protected final Hive db; protected final HiveConf conf; protected List> rootTasks; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index 4e58ad8..f3c01a8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -53,6 +53,7 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Index; import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.metastore.api.SkewedInfo; import org.apache.hadoop.hive.ql.Driver; @@ -214,24 +215,6 @@ public static String getTypeName(ASTNode node) throws SemanticException { return typeName; } - static class TablePartition { - String tableName; - HashMap partSpec = null; - - public TablePartition() { - } - - public TablePartition(ASTNode tblPart) throws SemanticException { - tableName = getDotName((getQualifiedTableName((ASTNode) tblPart.getChild(0)))); - if (tblPart.getChildCount() > 1) { - ASTNode part = (ASTNode) tblPart.getChild(1); - if (part.getToken().getType() == HiveParser.TOK_PARTSPEC) { - this.partSpec = DDLSemanticAnalyzer.getPartSpec(part); - } - } - } - } - public DDLSemanticAnalyzer(HiveConf conf) throws SemanticException { this(conf, createHiveDB(conf)); } @@ -1034,7 +1017,7 @@ private void analyzeTruncateTable(ASTNode ast) throws SemanticException { rootTasks.add(truncateTask); } - private boolean isFullSpec(Table table, Map partSpec) { + public static boolean isFullSpec(Table table, Map partSpec) { for (FieldSchema partCol : table.getPartCols()) { if (partSpec.get(partCol.getName()) == null) { return false; @@ -1139,20 +1122,25 @@ private void analyzeDropIndex(ASTNode ast) throws SemanticException { // configured not to ignore this boolean throwException = !ifExists && !HiveConf.getBoolVar(conf, ConfVars.DROPIGNORESNONEXISTENT); - if (throwException) { - try { - Index idx = db.getIndex(tableName, indexName); - } catch (HiveException e) { + Table tbl = getTable(tableName, false); + if (throwException && tbl == null) { + throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(tableName)); + } + try { + Index idx = db.getIndex(tableName, indexName); + } catch (HiveException e) { + if (!(e.getCause() instanceof NoSuchObjectException)) { + throw new SemanticException(ErrorMsg.GENERIC_ERROR.getMsg("dropping index"), e); + } + if (throwException) { throw new SemanticException(ErrorMsg.INVALID_INDEX.getMsg(indexName)); } } - - Table tbl = getTable(tableName, false); if (tbl != null) { - inputs.add(new ReadEntity(getTable(tableName))); + inputs.add(new ReadEntity(tbl)); } - DropIndexDesc dropIdxDesc = new DropIndexDesc(indexName, tableName); + DropIndexDesc dropIdxDesc = new DropIndexDesc(indexName, tableName, throwException); rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), dropIdxDesc), conf)); } @@ -1399,11 +1387,22 @@ private void addInputsOutputsAlterTable(String tableName, Map pa // ReadEntity as no lock. re.noLockNeeded(); inputs.add(re); - if (desc == null || desc.getOp() != AlterTableDesc.AlterTableTypes.ALTERPROTECTMODE) { + + if (isFullSpec(tab, partSpec)) { + // Fully specified partition spec Partition part = getPartition(tab, partSpec, true); - outputs.add(new WriteEntity(part, writeType)); - } - else { + outputs.add(new WriteEntity(part, writeType)); + } else { + // Partial partition spec supplied. Make sure this is allowed. + if (desc == null + || !AlterTableDesc.doesAlterTableTypeSupportPartialPartitionSpec(desc.getOp())) { + String alterTabletype = (desc != null) ? desc.getOp().name() : ""; + throw new SemanticException( + ErrorMsg.ALTER_TABLE_TYPE_PARTIAL_PARTITION_SPEC_NO_SUPPORTED, alterTabletype); + } else if (!conf.getBoolVar(HiveConf.ConfVars.DYNAMICPARTITIONING)) { + throw new SemanticException(ErrorMsg.DYNAMIC_PARTITION_DISABLED); + } + for (Partition part : getPartitions(tab, partSpec, true)) { outputs.add(new WriteEntity(part, writeType)); } @@ -2240,6 +2239,10 @@ private void analyzeShowFunctions(ASTNode ast) throws SemanticException { if (ast.getChildCount() == 1) { String funcNames = stripQuotes(ast.getChild(0).getText()); showFuncsDesc = new ShowFunctionsDesc(ctx.getResFile(), funcNames); + } else if (ast.getChildCount() == 2) { + assert (ast.getChild(0).getType() == HiveParser.KW_LIKE); + String funcNames = stripQuotes(ast.getChild(1).getText()); + showFuncsDesc = new ShowFunctionsDesc(ctx.getResFile(), funcNames, true); } else { showFuncsDesc = new ShowFunctionsDesc(ctx.getResFile()); } @@ -2805,7 +2808,7 @@ private Partition getPartitionForOutput(Table tab, Map currentPa * @param ast * The parsed command tree. * @throws SemanticException - * Parsin failed + * Parsing failed */ private void analyzeAlterTableTouch(String[] qualified, CommonTree ast) throws SemanticException { @@ -2929,8 +2932,8 @@ private void analyzeMetastoreCheck(CommonTree ast) throws SemanticException { * * @param ast Tree to extract partitions from. * @param tab Table. - * @param result Map of partitions by prefix length. Most of the time prefix length will - * be the same for all partition specs, so we can just OR the expressions. + * @return Map of partitions by prefix length. Most of the time prefix length will + * be the same for all partition specs, so we can just OR the expressions. */ private Map> getFullPartitionSpecs( CommonTree ast, Table tab, boolean canGroupExprs) throws SemanticException { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java index f2723ec..627ca7b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java @@ -57,6 +57,8 @@ import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; +import static org.apache.hadoop.hive.ql.plan.ReduceSinkDesc.ReducerTraits.AUTOPARALLEL; + /** * GenTezUtils is a collection of shared helper methods to produce * TezWork @@ -117,7 +119,7 @@ public ReduceWork createReduceWork(GenTezProcContext context, Operator root, reduceWork.setNumReduceTasks(reduceSink.getConf().getNumReducers()); - if (isAutoReduceParallelism && reduceSink.getConf().isAutoParallel()) { + if (isAutoReduceParallelism && reduceSink.getConf().getReducerTraits().contains(AUTOPARALLEL)) { reduceWork.setAutoReduceParallelism(true); // configured limit for reducers diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java index 516e576..364e8c9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java @@ -123,6 +123,11 @@ public Object process(Node nd, Stack stack, context.rootToWorkMap.put(root, work); } + // this is where we set the sort columns that we will be using for KeyValueInputMerge + if (operator instanceof DummyStoreOperator) { + work.addSortCols(root.getOpTraits().getSortCols().get(0)); + } + if (!context.childToWorkMap.containsKey(operator)) { List workItems = new LinkedList(); workItems.add(work); @@ -137,17 +142,18 @@ public Object process(Node nd, Stack stack, // we are currently walking the big table side of the merge join. we need to create or hook up // merge join work. MergeJoinWork mergeJoinWork = null; - if (context.opMergeJoinWorkMap.containsKey(operator)) { + if (context.opMergeJoinWorkMap.containsKey(context.currentMergeJoinOperator)) { // we have found a merge work corresponding to this closing operator. Hook up this work. - mergeJoinWork = context.opMergeJoinWorkMap.get(operator); + mergeJoinWork = context.opMergeJoinWorkMap.get(context.currentMergeJoinOperator); } else { // we need to create the merge join work mergeJoinWork = new MergeJoinWork(); mergeJoinWork.setMergeJoinOperator(context.currentMergeJoinOperator); tezWork.add(mergeJoinWork); - context.opMergeJoinWorkMap.put(operator, mergeJoinWork); + context.opMergeJoinWorkMap.put(context.currentMergeJoinOperator, mergeJoinWork); } // connect the work correctly. + work.addSortCols(root.getOpTraits().getSortCols().get(0)); mergeJoinWork.addMergedWork(work, null); Operator parentOp = getParentFromStack(context.currentMergeJoinOperator, stack); @@ -334,10 +340,16 @@ public Object process(Node nd, Stack stack, UnionWork unionWork = (UnionWork) followingWork; int index = getMergeIndex(tezWork, unionWork, rs); // guaranteed to be instance of MergeJoinWork if index is valid - MergeJoinWork mergeJoinWork = (MergeJoinWork) tezWork.getChildren(unionWork).get(index); - // disconnect the connection to union work and connect to merge work - followingWork = mergeJoinWork; - rWork = (ReduceWork) mergeJoinWork.getMainWork(); + BaseWork baseWork = tezWork.getChildren(unionWork).get(index); + if (baseWork instanceof MergeJoinWork) { + MergeJoinWork mergeJoinWork = (MergeJoinWork) baseWork; + // disconnect the connection to union work and connect to merge work + followingWork = mergeJoinWork; + rWork = (ReduceWork) mergeJoinWork.getMainWork(); + } else { + throw new SemanticException("Unknown work type found: " + + baseWork.getClass().getCanonicalName()); + } } else { rWork = (ReduceWork) followingWork; } @@ -391,6 +403,8 @@ private int getMergeIndex(TezWork tezWork, UnionWork unionWork, ReduceSinkOperat } else { index++; } + } else { + index++; } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index ef6d6f7..c903e8f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -1334,7 +1334,7 @@ showStatement | KW_SHOW KW_TABLES ((KW_FROM|KW_IN) db_name=identifier)? (KW_LIKE showStmtIdentifier|showStmtIdentifier)? -> ^(TOK_SHOWTABLES (TOK_FROM $db_name)? showStmtIdentifier?) | KW_SHOW KW_COLUMNS (KW_FROM|KW_IN) tableName ((KW_FROM|KW_IN) db_name=identifier)? -> ^(TOK_SHOWCOLUMNS tableName $db_name?) - | KW_SHOW KW_FUNCTIONS showFunctionIdentifier? -> ^(TOK_SHOWFUNCTIONS showFunctionIdentifier?) + | KW_SHOW KW_FUNCTIONS (KW_LIKE showFunctionIdentifier|showFunctionIdentifier)? -> ^(TOK_SHOWFUNCTIONS KW_LIKE? showFunctionIdentifier?) | KW_SHOW KW_PARTITIONS tabName=tableName partitionSpec? -> ^(TOK_SHOWPARTITIONS $tabName partitionSpec?) | KW_SHOW KW_CREATE KW_TABLE tabName=tableName -> ^(TOK_SHOW_CREATETABLE $tabName) | KW_SHOW KW_TABLE KW_EXTENDED ((KW_FROM|KW_IN) db_name=identifier)? KW_LIKE showStmtIdentifier partitionSpec? diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContext.java index 8694d07..b78ce90 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContext.java @@ -57,4 +57,12 @@ public String getUserName(); public void setUserName(String userName); + + public String getIpAddress(); + + public void setIpAddress(String ipAddress); + + public String getCommand(); + + public void setCommand(String command); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java index 4f6dad4..4ce705b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java @@ -33,6 +33,8 @@ Set inputs = null; Set outputs = null; private String userName; + private String ipAddress; + private String command; @Override public Hive getHive() throws HiveException { @@ -73,4 +75,24 @@ public String getUserName() { public void setUserName(String userName) { this.userName = userName; } + + @Override + public String getIpAddress() { + return ipAddress; + } + + @Override + public void setIpAddress(String ipAddress) { + this.ipAddress = ipAddress; + } + + @Override + public String getCommand() { + return command; + } + + @Override + public void setCommand(String command) { + this.command = command; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java index b3c4b47..b15a46d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java @@ -130,6 +130,10 @@ public static String getAppendedAliasFromId(String outer_id, String alias) { return (outer_id == null ? alias : outer_id + ":" + alias); } + public String getAlias() { + return qbp.getAlias(); + } + public QBParseInfo getParseInfo() { return qbp; } @@ -248,6 +252,12 @@ public boolean getIsQuery() { return isQuery; } + // to decide whether to rewrite RR of subquery + public boolean isTopLevelSelectStarQuery() { + return !isCTAS() && qbp.isTopLevelSimpleSelectStarQuery(); + } + + // to find target for fetch task conversion optimizer (not allows subqueries) public boolean isSimpleSelectQuery() { return qbp.isSimpleSelectQuery() && aliasToSubq.isEmpty() && !isCTAS() && !qbp.isAnalyzeCommand(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java index 86e4602..02c4be9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java @@ -27,6 +27,7 @@ import java.util.Map; import java.util.Set; +import org.antlr.runtime.tree.Tree; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.tableSpec; @@ -449,39 +450,49 @@ public void setOuterQueryLimit(int outerQueryLimit) { this.outerQueryLimit = outerQueryLimit; } + public boolean isTopLevelSimpleSelectStarQuery() { + if (alias != null || destToSelExpr.size() != 1 || !isSimpleSelectQuery()) { + return false; + } + for (ASTNode selExprs : destToSelExpr.values()) { + if (selExprs.getChildCount() != 1) { + return false; + } + Tree sel = selExprs.getChild(0).getChild(0); + if (sel == null || sel.getType() != HiveParser.TOK_ALLCOLREF) { + return false; + } + } + return true; + } + public boolean isSimpleSelectQuery() { - if (isSubQ || (joinExpr != null) - || (!destToGroupby.isEmpty()) || (!destToClusterby.isEmpty()) - || (!aliasToLateralViews.isEmpty())) { + if (isSubQ || joinExpr != null || !destToOrderby.isEmpty() || !destToSortby.isEmpty() + || !destToGroupby.isEmpty() || !destToClusterby.isEmpty() || !destToDistributeby.isEmpty() + || !aliasToLateralViews.isEmpty() || !destToLateralView.isEmpty()) { return false; } - Iterator>> aggrIter = destToAggregationExprs - .entrySet().iterator(); - while (aggrIter.hasNext()) { - HashMap h = aggrIter.next().getValue(); - if ((h != null) && (!h.isEmpty())) { + for (Map entry : destToAggregationExprs.values()) { + if (entry != null && !entry.isEmpty()) { return false; } } - if (!destToDistinctFuncExprs.isEmpty()) { - Iterator>> distn = destToDistinctFuncExprs - .entrySet().iterator(); - while (distn.hasNext()) { - List ct = distn.next().getValue(); - if (!ct.isEmpty()) { - return false; - } + for (Map entry : destToWindowingExprs.values()) { + if (entry != null && !entry.isEmpty()) { + return false; + } + } + + for (List ct : destToDistinctFuncExprs.values()) { + if (!ct.isEmpty()) { + return false; } } - Iterator> iter = nameToDest.entrySet() - .iterator(); - while (iter.hasNext()) { - Map.Entry entry = iter.next(); - ASTNode v = entry.getValue(); - if (!(((ASTNode)v.getChild(0)).getToken().getType() == HiveParser.TOK_TMP_FILE)) { + for (ASTNode v : nameToDest.values()) { + if (!(v.getChild(0).getType() == HiveParser.TOK_TMP_FILE)) { return false; } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java index 9c55379..469dc9f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java @@ -29,7 +29,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.exec.RowSchema; @@ -120,7 +119,11 @@ public boolean addMappingOnly(String tab_alias, String col_alias, ColumnInfo col f_map = new LinkedHashMap(); rslvMap.put(tab_alias, f_map); } - f_map.put(col_alias, colInfo); + ColumnInfo oldColInfo = f_map.put(col_alias, colInfo); + if (oldColInfo != null) { + LOG.warn("Duplicate column info for " + tab_alias + "." + col_alias + + " was overwritten in RowResolver map: " + oldColInfo + " by " + colInfo); + } String[] qualifiedAlias = new String[2]; qualifiedAlias[0] = tab_alias; @@ -195,17 +198,6 @@ public ColumnInfo get(String tab_alias, String col_alias) throws SemanticExcepti return ret; } - /** - * check if column name is already exist in RR - */ - public void checkColumn(String tableAlias, String columnAlias) throws SemanticException { - ColumnInfo prev = get(null, columnAlias); - if (prev != null && - (tableAlias == null || !tableAlias.equalsIgnoreCase(prev.getTabAlias()))) { - throw new SemanticException(ErrorMsg.AMBIGUOUS_COLUMN.getMsg(columnAlias)); - } - } - public ArrayList getColumnInfos() { return rowSchema.getSignature(); } @@ -351,40 +343,44 @@ public void setExpressionMap(Map expressionMap) { this.expressionMap = expressionMap; } + private static class IntRef { + public int val = 0; + } + + public static boolean add(RowResolver rrToAddTo, RowResolver rrToAddFrom, int numColumns) + throws SemanticException { + return add(rrToAddTo, rrToAddFrom, null, numColumns); + } - // TODO: 1) How to handle collisions? 2) Should we be cloning ColumnInfo or - // not? - public static int add(RowResolver rrToAddTo, RowResolver rrToAddFrom, - int outputColPos, int numColumns) throws SemanticException { + // TODO: 1) How to handle collisions? 2) Should we be cloning ColumnInfo or not? + private static boolean add(RowResolver rrToAddTo, RowResolver rrToAddFrom, + IntRef outputColPosRef, int numColumns) throws SemanticException { + boolean hasDuplicates = false; String tabAlias; String colAlias; String[] qualifiedColName; int i = 0; + int outputColPos = outputColPosRef == null ? 0 : outputColPosRef.val; for (ColumnInfo cInfoFrmInput : rrToAddFrom.getRowSchema().getSignature()) { if ( numColumns >= 0 && i == numColumns ) { break; } ColumnInfo newCI = null; - qualifiedColName = rrToAddFrom.getInvRslvMap().get( - cInfoFrmInput.getInternalName()); + String internalName = cInfoFrmInput.getInternalName(); + qualifiedColName = rrToAddFrom.reverseLookup(internalName); tabAlias = qualifiedColName[0]; colAlias = qualifiedColName[1]; newCI = new ColumnInfo(cInfoFrmInput); - newCI.setInternalName(SemanticAnalyzer - .getColumnInternalName(outputColPos)); + newCI.setInternalName(SemanticAnalyzer.getColumnInternalName(outputColPos)); outputColPos++; - if (rrToAddTo.get(tabAlias, colAlias) != null) { - LOG.debug("Found duplicate column alias in RR: " + rrToAddTo.get(tabAlias, colAlias)); - } else { - rrToAddTo.put(tabAlias, colAlias, newCI); - } + boolean isUnique = rrToAddTo.putWithCheck(tabAlias, colAlias, internalName, newCI); + hasDuplicates |= (!isUnique); - qualifiedColName = rrToAddFrom.getAlternateMappings(cInfoFrmInput - .getInternalName()); + qualifiedColName = rrToAddFrom.getAlternateMappings(internalName); if (qualifiedColName != null) { tabAlias = qualifiedColName[0]; colAlias = qualifiedColName[1]; @@ -393,31 +389,73 @@ public static int add(RowResolver rrToAddTo, RowResolver rrToAddFrom, i++; } - return outputColPos; - } - - public static int add(RowResolver rrToAddTo, RowResolver rrToAddFrom, - int outputColPos) throws SemanticException { - return add(rrToAddTo, rrToAddFrom, outputColPos, -1); - } - - /** - * Return a new row resolver that is combination of left RR and right RR. - * The schema will be schema of left, schema of right - * - * @param leftRR - * @param rightRR - * @return - * @throws SemanticException - */ - public static RowResolver getCombinedRR(RowResolver leftRR, - RowResolver rightRR) throws SemanticException { - int outputColPos = 0; - - RowResolver combinedRR = new RowResolver(); - outputColPos = add(combinedRR, leftRR, outputColPos); - outputColPos = add(combinedRR, rightRR, outputColPos); - - return combinedRR; - } + if (outputColPosRef != null) { + outputColPosRef.val = outputColPos; + } + return !hasDuplicates; + } + + /** + * Adds column to RR, checking for duplicate columns. Needed because CBO cannot handle the Hive + * behavior of blindly overwriting old mapping in RR and still somehow working after that. + * @return True if mapping was added without duplicates. + */ + public boolean putWithCheck(String tabAlias, String colAlias, + String internalName, ColumnInfo newCI) throws SemanticException { + ColumnInfo existing = get(tabAlias, colAlias); + // Hive adds the same mapping twice... I wish we could fix stuff like that. + if (existing == null) { + put(tabAlias, colAlias, newCI); + return true; + } else if (existing.isSameColumnForRR(newCI)) { + return true; + } + LOG.warn("Found duplicate column alias in RR: " + + existing.toMappingString(tabAlias, colAlias) + " adding " + + newCI.toMappingString(tabAlias, colAlias)); + if (internalName != null) { + existing = get(tabAlias, internalName); + if (existing == null) { + put(tabAlias, internalName, newCI); + return true; + } else if (existing.isSameColumnForRR(newCI)) { + return true; + } + LOG.warn("Failed to use internal name after finding a duplicate: " + + existing.toMappingString(tabAlias, internalName)); + } + return false; + } + + private static boolean add(RowResolver rrToAddTo, RowResolver rrToAddFrom, + IntRef outputColPosRef) throws SemanticException { + return add(rrToAddTo, rrToAddFrom, outputColPosRef, -1); + } + + public static boolean add(RowResolver rrToAddTo, RowResolver rrToAddFrom) + throws SemanticException { + return add(rrToAddTo, rrToAddFrom, null, -1); + } + + /** + * Return a new row resolver that is combination of left RR and right RR. + * The schema will be schema of left, schema of right + * + * @param leftRR + * @param rightRR + * @return + * @throws SemanticException + */ + public static RowResolver getCombinedRR(RowResolver leftRR, + RowResolver rightRR) throws SemanticException { + RowResolver combinedRR = new RowResolver(); + IntRef outputColPos = new IntRef(); + if (!add(combinedRR, leftRR, outputColPos)) { + LOG.warn("Duplicates detected when adding columns to RR: see previous message"); + } + if (!add(combinedRR, rightRR, outputColPos)) { + LOG.warn("Duplicates detected when adding columns to RR: see previous message"); + } + return combinedRR; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 19110ce..3afc071 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -22,6 +22,9 @@ import java.io.IOException; import java.io.Serializable; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.UndeclaredThrowableException; import java.math.BigDecimal; import java.util.ArrayList; import java.util.Arrays; @@ -30,6 +33,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -37,11 +41,11 @@ import java.util.TreeSet; import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; import com.google.common.annotations.VisibleForTesting; + import net.hydromatic.optiq.SchemaPlus; import net.hydromatic.optiq.tools.Frameworks; @@ -69,7 +73,6 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Order; -import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.QueryProperties; import org.apache.hadoop.hive.ql.exec.AbstractMapJoinOperator; @@ -90,6 +93,7 @@ import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; import org.apache.hadoop.hive.ql.exec.RowSchema; import org.apache.hadoop.hive.ql.exec.SMBMapJoinOperator; +import org.apache.hadoop.hive.ql.exec.SelectOperator; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; @@ -274,6 +278,7 @@ import org.eigenbase.rex.RexBuilder; import org.eigenbase.rex.RexInputRef; import org.eigenbase.rex.RexNode; +import org.eigenbase.rex.RexUtil; import org.eigenbase.rex.RexWindowBound; import org.eigenbase.rex.RexFieldCollation; import org.eigenbase.sql.SqlAggFunction; @@ -365,7 +370,7 @@ //flag for partial scan during analyze ... compute statistics protected boolean partialscan; - private volatile boolean runCBO = true; + private volatile boolean runCBO = true; // TODO: why is this volatile? private volatile boolean disableJoinMerge = false; /* @@ -377,6 +382,9 @@ */ private ArrayList ctesExpanded; + /** Not thread-safe. */ + private final ASTSearcher astSearcher = new ASTSearcher(); + private static class Phase1Ctx { String dest; int nextNum; @@ -509,7 +517,7 @@ public void doPhase1QBExpr(ASTNode ast, QBExpr qbexpr, String id, String alias) case HiveParser.TOK_QUERY: { QB qb = new QB(id, alias, true); Phase1Ctx ctx_1 = initPhase1Ctx(); - doPhase1(ast, qb, ctx_1); + doPhase1(ast, qb, ctx_1, null); qbexpr.setOpcode(QBExpr.Opcode.NULLOP); qbexpr.setQB(qb); @@ -1174,6 +1182,30 @@ private String processLateralView(QB qb, ASTNode lateralView) return alias; } + /** The context that doPhase1 uses to populate information pertaining + * to CBO (currently, this is used for CTAS and insert-as-select). */ + private static class PreCboCtx { + enum Type { + NONE, + INSERT, + CTAS, + + UNEXPECTED + } + public ASTNode nodeOfInterest; + public Type type = Type.NONE; + public void set(Type type, ASTNode ast) { + if (this.type != Type.NONE) { + STATIC_LOG.warn("Setting " + type + " when already " + this.type + + "; node " + ast.dump() + " vs old node " + nodeOfInterest.dump()); + this.type = Type.UNEXPECTED; + return; + } + this.type = type; + this.nodeOfInterest = ast; + } + } + /** * Phase 1: (including, but not limited to): * @@ -1191,7 +1223,7 @@ private String processLateralView(QB qb, ASTNode lateralView) * @throws SemanticException */ @SuppressWarnings({"fallthrough", "nls"}) - public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1) + public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1, PreCboCtx cboCtx) throws SemanticException { boolean phase1Result = true; @@ -1235,25 +1267,32 @@ public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1) String currentDatabase = SessionState.get().getCurrentDatabase(); String tab_name = getUnescapedName((ASTNode) ast.getChild(0).getChild(0), currentDatabase); qbp.addInsertIntoTable(tab_name); - // TODO: is this supposed to fall thru? case HiveParser.TOK_DESTINATION: ctx_1.dest = "insclause-" + ctx_1.nextNum; ctx_1.nextNum++; + boolean isTmpFileDest = false; + if (ast.getChildCount() > 0 && ast.getChild(0) instanceof ASTNode) { + ASTNode ch = (ASTNode)ast.getChild(0); + if (ch.getToken().getType() == HiveParser.TOK_DIR + && ch.getChildCount() > 0 && ch.getChild(0) instanceof ASTNode) { + ch = (ASTNode)ch.getChild(0); + isTmpFileDest = ch.getToken().getType() == HiveParser.TOK_TMP_FILE; + } + } // is there a insert in the subquery - if (qbp.getIsSubQ()) { - ASTNode ch = (ASTNode) ast.getChild(0); - if ((ch.getToken().getType() != HiveParser.TOK_DIR) - || (((ASTNode) ch.getChild(0)).getToken().getType() != HiveParser.TOK_TMP_FILE)) { - throw new SemanticException(ErrorMsg.NO_INSERT_INSUBQUERY - .getMsg(ast)); - } + if (qbp.getIsSubQ() && !isTmpFileDest) { + throw new SemanticException(ErrorMsg.NO_INSERT_INSUBQUERY.getMsg(ast)); + } + if (cboCtx != null && !isTmpFileDest) { + cboCtx.set(PreCboCtx.Type.INSERT, ast); } qbp.setDestForClause(ctx_1.dest, (ASTNode) ast.getChild(0)); - if (qbp.getClauseNamesForDest().size() > 1) + if (qbp.getClauseNamesForDest().size() > 1) { queryProperties.setMultiDestQuery(true); + } break; case HiveParser.TOK_FROM: @@ -1476,12 +1515,18 @@ public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1) int child_count = ast.getChildCount(); for (int child_pos = 0; child_pos < child_count && phase1Result; ++child_pos) { // Recurse - phase1Result = phase1Result && doPhase1((ASTNode) ast.getChild(child_pos), qb, ctx_1); + phase1Result = phase1Result && doPhase1( + (ASTNode)ast.getChild(child_pos), qb, ctx_1, cboCtx); } } return phase1Result; } + private void traceLogAst(ASTNode ast, String what) { + if (!LOG.isTraceEnabled()) return; + LOG.trace(what + ast.dump()); + } + private void getMetaData(QBExpr qbexpr, ReadEntity parentInput) throws SemanticException { if (qbexpr.getOpcode() == QBExpr.Opcode.NULLOP) { @@ -1756,6 +1801,7 @@ public void getMetaData(QB qb, ReadEntity parentInput) throws SemanticException qb.getParseInfo().addTableSpec(ts.tableName.toLowerCase(), ts); } } else { + // This is the only place where isQuery is set to true; it defaults to false. qb.setIsQuery(true); fname = ctx.getMRTmpPath().toString(); ctx.setResDir(new Path(fname)); @@ -2460,7 +2506,7 @@ private Operator genPlanForSubQueryPredicate( ISubQueryJoinInfo subQueryPredicate) throws SemanticException { qbSQ.setSubQueryDef(subQueryPredicate.getSubQuery()); Phase1Ctx ctx_1 = initPhase1Ctx(); - doPhase1(subQueryPredicate.getSubQueryAST(), qbSQ, ctx_1); + doPhase1(subQueryPredicate.getSubQueryAST(), qbSQ, ctx_1, null); getMetaData(qbSQ); Operator op = genPlan(qbSQ); return op; @@ -2659,7 +2705,10 @@ private Operator genNotNullFilterForJoinSourcePlan(QB qb, Operator input, if (joinKeys == null || joinKeys.length == 0) { return input; } - + Map hashes = new HashMap(); + if (input instanceof FilterOperator) { + ExprNodeDescUtils.getExprNodeColumnDesc(Arrays.asList(((FilterDesc)input.getConf()).getPredicate()), hashes); + } ExprNodeDesc filterPred = null; List nullSafes = joinTree.getNullSafes(); for (int i = 0; i < joinKeys.length; i++) { @@ -2669,6 +2718,10 @@ private Operator genNotNullFilterForJoinSourcePlan(QB qb, Operator input, // virtual column, since those columns can never be null. continue; } + if(null != hashes.get(joinKeys[i].hashCode())) { + // there is already a predicate on this src. + continue; + } List args = new ArrayList(); args.add(joinKeys[i]); ExprNodeDesc nextExpr = ExprNodeGenericFuncDesc.newInstance( @@ -2706,9 +2759,10 @@ private Operator genNotNullFilterForJoinSourcePlan(QB qb, Operator input, } @SuppressWarnings("nls") - private Integer genColListRegex(String colRegex, String tabAlias, - ASTNode sel, ArrayList col_list, - RowResolver input, Integer pos, RowResolver output, List aliases, boolean subQuery) + // TODO: make aliases unique, otherwise needless rewriting takes place + private Integer genColListRegex(String colRegex, String tabAlias, ASTNode sel, + ArrayList col_list, HashSet excludeCols, RowResolver input, + Integer pos, RowResolver output, List aliases, boolean ensureUniqueCols) throws SemanticException { // The table alias should exist @@ -2749,6 +2803,9 @@ private Integer genColListRegex(String colRegex, String tabAlias, // from the input schema for (Map.Entry entry : fMap.entrySet()) { ColumnInfo colInfo = entry.getValue(); + if (excludeCols != null && excludeCols.contains(colInfo)) { + continue; // This was added during plan generation. + } String name = colInfo.getInternalName(); String[] tmp = input.reverseLookup(name); @@ -2766,9 +2823,6 @@ private Integer genColListRegex(String colRegex, String tabAlias, continue; } - if (subQuery) { - output.checkColumn(tmp[0], tmp[1]); - } ColumnInfo oColInfo = inputColsProcessed.get(colInfo); if (oColInfo == null) { ExprNodeColumnDesc expr = new ExprNodeColumnDesc(colInfo.getType(), @@ -2780,7 +2834,14 @@ private Integer genColListRegex(String colRegex, String tabAlias, colInfo.getIsVirtualCol(), colInfo.isHiddenVirtualCol()); inputColsProcessed.put(colInfo, oColInfo); } - output.put(tmp[0], tmp[1], oColInfo); + if (ensureUniqueCols) { + if (!output.putWithCheck(tmp[0], tmp[1], null, oColInfo)) { + throw new OptiqSemanticException("Cannot add column to RR: " + tmp[0] + "." + tmp[1] + + " => " + oColInfo + " due to duplication, see previous warnings"); + } + } else { + output.put(tmp[0], tmp[1], oColInfo); + } pos = Integer.valueOf(pos.intValue() + 1); matched++; @@ -3394,7 +3455,6 @@ private static boolean isRegex(String pattern, HiveConf conf) { posn++; } - boolean subQuery = qb.getParseInfo().getIsSubQ(); boolean isInTransform = (selExprList.getChild(posn).getChild(0).getType() == HiveParser.TOK_TRANSFORM); if (isInTransform) { @@ -3432,7 +3492,7 @@ private static boolean isRegex(String pattern, HiveConf conf) { } if (isUDTF && (selectStar = udtfExprType == HiveParser.TOK_FUNCTIONSTAR)) { genColListRegex(".*", null, (ASTNode) udtfExpr.getChild(0), - col_list, inputRR, pos, out_rwsch, qb.getAliases(), subQuery); + col_list, null, inputRR, pos, out_rwsch, qb.getAliases(), false); } } @@ -3554,7 +3614,7 @@ private static boolean isRegex(String pattern, HiveConf conf) { if (expr.getType() == HiveParser.TOK_ALLCOLREF) { pos = genColListRegex(".*", expr.getChildCount() == 0 ? null : getUnescapedName((ASTNode) expr.getChild(0)).toLowerCase(), - expr, col_list, inputRR, pos, out_rwsch, qb.getAliases(), subQuery); + expr, col_list, null, inputRR, pos, out_rwsch, qb.getAliases(), false); selectStar = true; } else if (expr.getType() == HiveParser.TOK_TABLE_OR_COL && !hasAsClause && !inputRR.getIsExprResolver() @@ -3563,7 +3623,7 @@ private static boolean isRegex(String pattern, HiveConf conf) { // This can only happen without AS clause // We don't allow this for ExprResolver - the Group By case pos = genColListRegex(unescapeIdentifier(expr.getChild(0).getText()), - null, expr, col_list, inputRR, pos, out_rwsch, qb.getAliases(), subQuery); + null, expr, col_list, null, inputRR, pos, out_rwsch, qb.getAliases(), false); } else if (expr.getType() == HiveParser.DOT && expr.getChild(0).getType() == HiveParser.TOK_TABLE_OR_COL && inputRR.hasTableAlias(unescapeIdentifier(expr.getChild(0) @@ -3574,9 +3634,8 @@ private static boolean isRegex(String pattern, HiveConf conf) { // This can only happen without AS clause // We don't allow this for ExprResolver - the Group By case pos = genColListRegex(unescapeIdentifier(expr.getChild(1).getText()), - unescapeIdentifier(expr.getChild(0).getChild(0).getText() - .toLowerCase()), expr, col_list, inputRR, pos, out_rwsch, - qb.getAliases(), subQuery); + unescapeIdentifier(expr.getChild(0).getChild(0).getText().toLowerCase()), + expr, col_list, null, inputRR, pos, out_rwsch, qb.getAliases(), false); } else { // Case when this is an expression TypeCheckCtx tcCtx = new TypeCheckCtx(inputRR); @@ -3590,9 +3649,6 @@ private static boolean isRegex(String pattern, HiveConf conf) { colAlias = recommended; } col_list.add(exp); - if (subQuery) { - out_rwsch.checkColumn(tabAlias, colAlias); - } ColumnInfo colInfo = new ColumnInfo(getColumnInternalName(pos), exp.getWritableObjectInspector(), tabAlias, false); @@ -6183,8 +6239,7 @@ private Operator genFileSinkPlan(String dest, QB qb, Operator input) ArrayList colInfos = inputRR.getColumnInfos(); // CTAS case: the file output format and serde are defined by the create - // table command - // rather than taking the default value + // table command rather than taking the default value List field_schemas = null; CreateTableDesc tblDesc = qb.getTableDesc(); if (tblDesc != null) { @@ -6205,7 +6260,8 @@ private Operator genFileSinkPlan(String dest, QB qb, Operator input) if (!("".equals(nm[0])) && nm[1] != null) { colName = unescapeIdentifier(colInfo.getAlias()).toLowerCase(); // remove `` } - col.setName(colName);; + String ctasColName = fixCtasColumnName(colName, colInfo, inputRR); + col.setName(ctasColName); col.setType(colInfo.getType().getTypeName()); field_schemas.add(col); } @@ -6383,6 +6439,14 @@ private Operator genFileSinkPlan(String dest, QB qb, Operator input) return output; } + private static String fixCtasColumnName(String colName, ColumnInfo colInfo, RowResolver rr) { + int lastDot = colName.lastIndexOf('.'); + if (lastDot < 0) return colName; // alias is not fully qualified + String nqColumnName = colName.substring(lastDot + 1); + STATIC_LOG.debug("Replacing " + colName + " (produced by CBO) by " + nqColumnName); + return nqColumnName; + } + // Check constraints on acid tables. This includes // * no insert overwrites // * no use of vectorization @@ -6403,7 +6467,8 @@ private void checkAcidConstraints(QB qb, TableDesc tableDesc, conf.setBoolVar(ConfVars.HIVE_VECTORIZATION_ENABLED, false); } LOG.info("Modifying config values for ACID write"); - conf.setBoolVar(ConfVars.HIVEOPTREDUCEDEDUPLICATION, false); + conf.setBoolVar(ConfVars.HIVEOPTREDUCEDEDUPLICATION, true); + conf.setIntVar(ConfVars.HIVEOPTREDUCEDEDUPLICATIONMINREDUCER, 1); conf.setBoolVar(ConfVars.HIVE_HADOOP_SUPPORTS_SUBDIRECTORIES, true); conf.set(AcidUtils.CONF_ACID_KEY, "true"); @@ -7695,7 +7760,9 @@ private QBJoinTree genUniqueJoinTree(QB qb, ASTNode joinParseTree, } joinTree.setJoinCond(condn); - if (qb.getParseInfo().getHints() != null) { + if ((qb.getParseInfo().getHints() != null) + && !(conf.getVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez"))) { + LOG.info("STREAMTABLE hint honored."); parseStreamTables(joinTree, qb); } @@ -7978,7 +8045,9 @@ private QBJoinTree genJoinTree(QB qb, ASTNode joinParseTree, joinTree.setMapAliases(mapAliases); - parseStreamTables(joinTree, qb); + if ((conf.getVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) == false) { + parseStreamTables(joinTree, qb); + } } return joinTree; @@ -8257,7 +8326,8 @@ private void mergeJoinTree(QB qb) { continue; } JoinType prevType = null; // save join type - for (int j = i - 1; j >= 0; j--) { + boolean continueScanning = true; + for (int j = i - 1; j >= 0 && continueScanning; j--) { QBJoinTree node = trees.get(j); if (node == null) { continue; @@ -8273,6 +8343,7 @@ private void mergeJoinTree(QB qb) { if (!node.getNoOuterJoin() || !target.getNoOuterJoin()) { if (node.getRightAliases().length + target.getRightAliases().length + 1 > 16) { LOG.info(ErrorMsg.JOINNODE_OUTERJOIN_MORETHAN_16); + continueScanning = !runCBO; continue; } } @@ -8280,6 +8351,11 @@ private void mergeJoinTree(QB qb) { trees.set(j, null); continue; // continue merging with next alias } + /* + * for CBO provided orderings, don't attempt to reorder joins. + * only convert consecutive joins into n-way joins. + */ + continueScanning = !runCBO; if (prevType == null) { prevType = currType; } @@ -8906,24 +8982,6 @@ private Operator genPostGroupByBodyPlan(Operator curr, String dest, QB qb, } } - // change curr ops row resolver's tab aliases to query alias if it - // exists - if (qb.getParseInfo().getAlias() != null) { - RowResolver rr = opParseCtx.get(curr).getRowResolver(); - RowResolver newRR = new RowResolver(); - String alias = qb.getParseInfo().getAlias(); - for (ColumnInfo colInfo : rr.getColumnInfos()) { - String name = colInfo.getInternalName(); - String[] tmp = rr.reverseLookup(name); - if ("".equals(tmp[0]) || tmp[1] == null) { - // ast expression is not a valid column name for table - tmp[1] = colInfo.getInternalName(); - } - newRR.put(alias, tmp[1], colInfo); - } - opParseCtx.get(curr).setRowResolver(newRR); - } - return curr; } @@ -9000,38 +9058,65 @@ private Operator genUnionPlan(String unionalias, String leftalias, rightOp = genInputSelectForUnion(rightOp, rightmap, rightalias, unionoutRR, unionalias); } - // If one of the children is a union, merge with it + // If one of the children (left or right) is: + // (i) a union, or + // (ii) an identity projection followed by a union, + // merge with it // else create a new one - if ((leftOp instanceof UnionOperator) || (rightOp instanceof UnionOperator)) { - if (leftOp instanceof UnionOperator) { - // make left a child of right - List> child = - new ArrayList>(); - child.add(leftOp); - rightOp.setChildOperators(child); - - List> parent = leftOp - .getParentOperators(); - parent.add(rightOp); - - UnionDesc uDesc = ((UnionOperator) leftOp).getConf(); - uDesc.setNumInputs(uDesc.getNumInputs() + 1); - return putOpInsertMap(leftOp, unionoutRR); - } else { - // make right a child of left - List> child = - new ArrayList>(); - child.add(rightOp); - leftOp.setChildOperators(child); + if (leftOp instanceof UnionOperator || + (leftOp instanceof SelectOperator && + leftOp.getParentOperators() != null && + !leftOp.getParentOperators().isEmpty() && + leftOp.getParentOperators().get(0) instanceof UnionOperator && + ((SelectOperator)leftOp).isIdentitySelect()) ) { + + if(!(leftOp instanceof UnionOperator)) { + Operator oldChild = leftOp; + leftOp = (Operator) leftOp.getParentOperators().get(0); + leftOp.removeChildAndAdoptItsChildren(oldChild); + } + + // make left a child of right + List> child = + new ArrayList>(); + child.add(leftOp); + rightOp.setChildOperators(child); - List> parent = rightOp - .getParentOperators(); - parent.add(leftOp); - UnionDesc uDesc = ((UnionOperator) rightOp).getConf(); - uDesc.setNumInputs(uDesc.getNumInputs() + 1); + List> parent = leftOp + .getParentOperators(); + parent.add(rightOp); - return putOpInsertMap(rightOp, unionoutRR); + UnionDesc uDesc = ((UnionOperator) leftOp).getConf(); + uDesc.setNumInputs(uDesc.getNumInputs() + 1); + return putOpInsertMap(leftOp, unionoutRR); + } + + if (rightOp instanceof UnionOperator || + (rightOp instanceof SelectOperator && + rightOp.getParentOperators() != null && + !rightOp.getParentOperators().isEmpty() && + rightOp.getParentOperators().get(0) instanceof UnionOperator && + ((SelectOperator)rightOp).isIdentitySelect()) ) { + + if(!(rightOp instanceof UnionOperator)) { + Operator oldChild = rightOp; + rightOp = (Operator) rightOp.getParentOperators().get(0); + rightOp.removeChildAndAdoptItsChildren(oldChild); } + + // make right a child of left + List> child = + new ArrayList>(); + child.add(rightOp); + leftOp.setChildOperators(child); + + List> parent = rightOp + .getParentOperators(); + parent.add(leftOp); + UnionDesc uDesc = ((UnionOperator) rightOp).getConf(); + uDesc.setNumInputs(uDesc.getNumInputs() + 1); + + return putOpInsertMap(rightOp, unionoutRR); } // Create a new union operator @@ -9508,13 +9593,14 @@ private void setupStats(TableScanDesc tsDesc, QBParseInfo qbp, Table tab, String } } - private Operator genPlan(QBExpr qbexpr) throws SemanticException { + private Operator genPlan(QB parent, QBExpr qbexpr) throws SemanticException { if (qbexpr.getOpcode() == QBExpr.Opcode.NULLOP) { - return genPlan(qbexpr.getQB()); + boolean skipAmbiguityCheck = viewSelect == null && parent.isTopLevelSelectStarQuery(); + return genPlan(qbexpr.getQB(), skipAmbiguityCheck); } if (qbexpr.getOpcode() == QBExpr.Opcode.UNION) { - Operator qbexpr1Ops = genPlan(qbexpr.getQBExpr1()); - Operator qbexpr2Ops = genPlan(qbexpr.getQBExpr2()); + Operator qbexpr1Ops = genPlan(parent, qbexpr.getQBExpr1()); + Operator qbexpr2Ops = genPlan(parent, qbexpr.getQBExpr2()); return genUnionPlan(qbexpr.getAlias(), qbexpr.getQBExpr1().getAlias(), qbexpr1Ops, qbexpr.getQBExpr2().getAlias(), qbexpr2Ops); @@ -9522,8 +9608,13 @@ private Operator genPlan(QBExpr qbexpr) throws SemanticException { return null; } - @SuppressWarnings("nls") public Operator genPlan(QB qb) throws SemanticException { + return genPlan(qb, false); + } + + @SuppressWarnings("nls") + public Operator genPlan(QB qb, boolean skipAmbiguityCheck) + throws SemanticException { // First generate all the opInfos for the elements in the from clause Map aliasToOpInfo = new HashMap(); @@ -9531,8 +9622,7 @@ public Operator genPlan(QB qb) throws SemanticException { // Recurse over the subqueries to fill the subquery part of the plan for (String alias : qb.getSubqAliases()) { QBExpr qbexpr = qb.getSubqForAlias(alias); - aliasToOpInfo.put(alias, genPlan(qbexpr)); - qbexpr.setAlias(alias); + aliasToOpInfo.put(alias, genPlan(qb, qbexpr)); } // Recurse over all the source tables @@ -9631,10 +9721,38 @@ public Operator genPlan(QB qb) throws SemanticException { LOG.debug("Created Plan for Query Block " + qb.getId()); } + if (qb.getAlias() != null) { + rewriteRRForSubQ(qb.getAlias(), bodyOpInfo, skipAmbiguityCheck); + } + this.qb = qb; return bodyOpInfo; } + // change curr ops row resolver's tab aliases to subq alias + private void rewriteRRForSubQ(String alias, Operator operator, boolean skipAmbiguityCheck) + throws SemanticException { + RowResolver rr = opParseCtx.get(operator).getRowResolver(); + RowResolver newRR = new RowResolver(); + for (ColumnInfo colInfo : rr.getColumnInfos()) { + String name = colInfo.getInternalName(); + String[] tmp = rr.reverseLookup(name); + if ("".equals(tmp[0]) || tmp[1] == null) { + // ast expression is not a valid column name for table + tmp[1] = colInfo.getInternalName(); + } else if (newRR.get(alias, tmp[1]) != null) { + // enforce uniqueness of column names + if (!skipAmbiguityCheck) { + throw new SemanticException(ErrorMsg.AMBIGUOUS_COLUMN.getMsg(tmp[1] + " in " + alias)); + } + // if it's wrapped by top-level select star query, skip ambiguity check (for backward compatibility) + tmp[1] = colInfo.getInternalName(); + } + newRR.put(alias, tmp[1], colInfo); + } + opParseCtx.get(operator).setRowResolver(newRR); + } + private Table getDummyTable() throws SemanticException { Path dummyPath = createDummyFile(); Table desc = new Table(DUMMY_DATABASE, DUMMY_TABLE); @@ -9718,14 +9836,25 @@ private Operator genLateralViewPlan(QB qb, Operator op, ASTNode lateralViewTree) throws SemanticException { RowResolver lvForwardRR = new RowResolver(); RowResolver source = opParseCtx.get(op).getRowResolver(); + Map lvfColExprMap = new HashMap(); + Map selColExprMap = new HashMap(); + List colList = new ArrayList(); + List colNames = new ArrayList(); for (ColumnInfo col : source.getColumnInfos()) { String[] tabCol = source.reverseLookup(col.getInternalName()); lvForwardRR.put(tabCol[0], tabCol[1], col); + ExprNodeDesc colExpr = new ExprNodeColumnDesc(col.getType(), col.getInternalName(), + col.getTabAlias(), false); + colList.add(colExpr); + colNames.add(colExpr.getName()); + lvfColExprMap.put(col.getInternalName(), colExpr); + selColExprMap.put(col.getInternalName(), colExpr.clone()); } Operator lvForward = putOpInsertMap(OperatorFactory.getAndMakeChild( new LateralViewForwardDesc(), new RowSchema(lvForwardRR.getColumnInfos()), op), lvForwardRR); + lvForward.setColumnExprMap(lvfColExprMap); // The order in which the two paths are added is important. The // lateral view join operator depends on having the select operator @@ -9734,9 +9863,12 @@ private Operator genLateralViewPlan(QB qb, Operator op, ASTNode lateralViewTree) // Get the all path by making a select(*). RowResolver allPathRR = opParseCtx.get(lvForward).getRowResolver(); // Operator allPath = op; + SelectDesc sDesc = new SelectDesc(colList, colNames, false); + sDesc.setSelStarNoCompute(true); Operator allPath = putOpInsertMap(OperatorFactory.getAndMakeChild( - new SelectDesc(true), new RowSchema(allPathRR.getColumnInfos()), + sDesc, new RowSchema(allPathRR.getColumnInfos()), lvForward), allPathRR); + allPath.setColumnExprMap(selColExprMap); int allColumns = allPathRR.getColumnInfos().size(); // Get the UDTF Path QB blankQb = new QB(null, null, false); @@ -9838,11 +9970,14 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { // analyze and process the position alias processPositionAlias(ast); + // Check configuration for CBO first. + runCBO = runCBO && HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_CBO_ENABLED); // analyze create table command + PreCboCtx cboCtx = runCBO ? new PreCboCtx() : null; if (ast.getToken().getType() == HiveParser.TOK_CREATETABLE) { // if it is not CTAS, we don't need to go further and just return - if ((child = analyzeCreateTable(ast, qb)) == null) { + if ((child = analyzeCreateTable(ast, qb, cboCtx)) == null) { return; } } else { @@ -9851,7 +9986,8 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { // analyze create view command if (ast.getToken().getType() == HiveParser.TOK_CREATEVIEW || - (ast.getToken().getType() == HiveParser.TOK_ALTERVIEW && ast.getChild(1).getType() == HiveParser.TOK_QUERY)) { + (ast.getToken().getType() == HiveParser.TOK_ALTERVIEW + && ast.getChild(1).getType() == HiveParser.TOK_QUERY)) { child = analyzeCreateView(ast, qb); SessionState.get().setCommandType(HiveOperation.CREATEVIEW); if (child == null) { @@ -9864,7 +10000,7 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { // continue analyzing from the child ASTNode. Phase1Ctx ctx_1 = initPhase1Ctx(); - if (!doPhase1(child, qb, ctx_1)) { + if (!doPhase1(child, qb, ctx_1, cboCtx)) { // if phase1Result false return return; } @@ -9874,19 +10010,16 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { getMetaData(qb); LOG.info("Completed getting MetaData in Semantic Analysis"); - + // Note: for now, we don't actually pass the queryForCbo to CBO, because it accepts qb, not + // AST, and can also access all the private stuff in SA. We rely on the fact that CBO + // ignores the unknown tokens (create table, destination), so if the query is otherwise ok, + // it is as if we did remove those and gave CBO the proper AST. That is kinda hacky. if (runCBO) { - boolean tokenTypeIsQuery = ast.getToken().getType() == HiveParser.TOK_QUERY - || ast.getToken().getType() == HiveParser.TOK_EXPLAIN; - if (!tokenTypeIsQuery || createVwDesc != null - || !HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_CBO_ENABLED) - || !canHandleQuery(qb, true) || !HiveOptiqUtil.validateASTForCBO(ast)) { - runCBO = false; - } - - if (runCBO) { - disableJoinMerge = true; + ASTNode queryForCbo = ast; + if (cboCtx.type == PreCboCtx.Type.CTAS) { + queryForCbo = cboCtx.nodeOfInterest; // nodeOfInterest is the query } + runCBO = canHandleAstForCbo(queryForCbo, qb, cboCtx); } // Save the result schema derived from the sink operator produced @@ -9896,6 +10029,7 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { Operator sinkOp = null; if (runCBO) { + disableJoinMerge = true; OptiqBasedPlanner optiqPlanner = new OptiqBasedPlanner(); boolean reAnalyzeAST = false; @@ -9903,10 +10037,17 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { // 1. Gen Optimized AST ASTNode newAST = optiqPlanner.getOptimizedAST(prunedPartitions); + // 1.1. Fix up the query for insert/ctas + newAST = fixUpCtasAndInsertAfterCbo(ast, newAST, cboCtx); + // 2. Regen OP plan from optimized AST init(false); + if (cboCtx.type == PreCboCtx.Type.CTAS) { + // Redo create-table analysis, because it's not part of doPhase1. + newAST = reAnalyzeCtasAfterCbo(newAST); + } ctx_1 = initPhase1Ctx(); - if (!doPhase1(newAST, qb, ctx_1)) { + if (!doPhase1(newAST, qb, ctx_1, null)) { throw new RuntimeException( "Couldn't do phase1 on CBO optimized query plan"); } @@ -9916,7 +10057,7 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { //prunedPartitions = ImmutableMap.copyOf(prunedPartitions); getMetaData(qb); - disableJoinMerge = true; + disableJoinMerge = false; sinkOp = genPlan(qb); LOG.info("CBO Succeeded; optimized logical plan."); LOG.debug(newAST.dump()); @@ -9929,13 +10070,21 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { * .getRowResolver(), true); */ } catch (Exception e) { - LOG.error("CBO failed, skipping CBO. ", e); - if (!conf.getBoolVar(ConfVars.HIVE_IN_TEST) || - (optiqPlanner.noColsMissingStats.get() > 0) || - e instanceof OptiqSemanticException) { + boolean isMissingStats = optiqPlanner.noColsMissingStats.get() > 0; + if (isMissingStats) { + LOG.error("CBO failed due to missing column stats (see previous errors), skipping CBO"); + } else { + LOG.error("CBO failed, skipping CBO. ", e); + } + if (!conf.getBoolVar(ConfVars.HIVE_IN_TEST) || isMissingStats + || e instanceof OptiqSemanticException) { reAnalyzeAST = true; + } else if (e instanceof SemanticException) { + throw (SemanticException)e; + } else if (e instanceof RuntimeException) { + throw (RuntimeException)e; } else { - throw e instanceof SemanticException ? (SemanticException) e : new SemanticException(e); + throw new SemanticException(e); } } finally { runCBO = false; @@ -10044,18 +10193,135 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { return; } + private ASTNode fixUpCtasAndInsertAfterCbo( + ASTNode originalAst, ASTNode newAst, PreCboCtx cboCtx) throws SemanticException { + switch (cboCtx.type) { + case NONE: return newAst; // nothing to do + case CTAS: { + // Patch the optimized query back into original CTAS AST, replacing the original query. + replaceASTChild(cboCtx.nodeOfInterest, newAst); + return originalAst; + } + case INSERT: { + // We need to patch the dest back to original into new query. + // This makes assumptions about the structure of the AST. + ASTNode newDest = astSearcher.simpleBreadthFirstSearch( + newAst, HiveParser.TOK_QUERY, HiveParser.TOK_INSERT, HiveParser.TOK_DESTINATION); + if (newDest == null) { + LOG.error("Cannot find destination after CBO; new ast is "+ newAst.dump()); + throw new SemanticException("Cannot find destination after CBO"); + } + replaceASTChild(newDest, cboCtx.nodeOfInterest); + return newAst; + } + default: throw new AssertionError("Unexpected type " + cboCtx.type); + } + } + + private ASTNode reAnalyzeCtasAfterCbo(ASTNode newAst) throws SemanticException { + // analyzeCreateTable uses this.ast, but doPhase1 doesn't, so only reset it here. + this.ast = newAst; + newAst = analyzeCreateTable(newAst, qb, null); + if (newAst == null) { + LOG.error("analyzeCreateTable failed to initialize CTAS after CBO;" + + " new ast is " + this.ast.dump()); + throw new SemanticException("analyzeCreateTable failed to initialize CTAS after CBO"); + } + return newAst; + } + + private boolean canHandleAstForCbo(ASTNode ast, QB qb, PreCboCtx cboCtx) { + int root = ast.getToken().getType(); + boolean needToLogMessage = LOG.isInfoEnabled(); + boolean isSupportedRoot = + root == HiveParser.TOK_QUERY || root == HiveParser.TOK_EXPLAIN || qb.isCTAS(); + // Check AST. + // Assumption: If top level QB is query then everything below it must also be Query + // Can there be an insert or CTAS that wouldn't + // be supported and would require additional checks similar to IsQuery? + boolean isSupportedType = + qb.getIsQuery() || qb.isCTAS() || cboCtx.type == PreCboCtx.Type.INSERT; + boolean result = isSupportedRoot && isSupportedType && createVwDesc == null; + if (!result) { + if (needToLogMessage) { + String msg = ""; + if (!isSupportedRoot) msg += "doesn't have QUERY or EXPLAIN as root and not a CTAS; "; + if (!isSupportedType) msg += "is not a query, CTAS, or insert; "; + if (createVwDesc != null) msg += "has create view; "; + + if (msg.isEmpty()) msg += "has some unspecified limitations; "; + LOG.info("Not invoking CBO because the statement " + msg.substring(0, msg.length() - 2)); + } + return false; + } + // Now check QB in more detail. canHandleQbForCbo returns null if query can be handled. + String msg = canHandleQbForCbo(qb, true, needToLogMessage); + if (msg == null) { + return true; + } + if (needToLogMessage) { + LOG.info("Not invoking CBO because the statement " + msg.substring(0, msg.length() - 2)); + } + return false; + } + + private class ASTSearcher { + private final LinkedList searchQueue = new LinkedList(); + /** + * Performs breadth-first search of the AST for a nested set of tokens. Tokens don't have to be + * each others' direct children, they can be separated by layers of other tokens. For each token + * in the list, the first one found is matched and there's no backtracking; thus, if AST has + * multiple instances of some token, of which only one matches, it is not guaranteed to be found. + * We use this for simple things. + * Not thread-safe - reuses searchQueue. + */ + public ASTNode simpleBreadthFirstSearch(ASTNode ast, int... tokens) { + searchQueue.clear(); + searchQueue.add(ast); + for (int i = 0; i < tokens.length; ++i) { + boolean found = false; + int token = tokens[i]; + while (!searchQueue.isEmpty() && !found) { + ASTNode next = searchQueue.poll(); + found = next.getType() == token; + if (found) { + if (i == tokens.length - 1) return next; + searchQueue.clear(); + } + for (int j = 0; j < next.getChildCount(); ++j) { + searchQueue.add((ASTNode)next.getChild(j)); + } + } + if (!found) return null; + } + return null; + } + } + + private void replaceASTChild(ASTNode child, ASTNode newChild) { + ASTNode parent = (ASTNode)child.parent; + int childIndex = child.childIndex; + parent.deleteChild(childIndex); + parent.insertChild(childIndex, newChild); + } + private void putAccessedColumnsToReadEntity(HashSet inputs, ColumnAccessInfo columnAccessInfo) { Map> tableToColumnAccessMap = columnAccessInfo.getTableToColumnAccessMap(); if (tableToColumnAccessMap != null && !tableToColumnAccessMap.isEmpty()) { for(ReadEntity entity: inputs) { + List cols; switch (entity.getType()) { case TABLE: - entity.getAccessedColumns().addAll( - tableToColumnAccessMap.get(entity.getTable().getCompleteName())); + cols = tableToColumnAccessMap.get(entity.getTable().getCompleteName()); + if (cols != null && !cols.isEmpty()) { + entity.getAccessedColumns().addAll(cols); + } break; case PARTITION: - entity.getAccessedColumns().addAll( - tableToColumnAccessMap.get(entity.getPartition().getTable().getCompleteName())); + cols = tableToColumnAccessMap.get(entity.getPartition().getTable().getCompleteName()); + if (cols != null && !cols.isEmpty()) { + entity.getAccessedColumns().addAll(cols); + } break; default: // no-op @@ -10535,8 +10801,8 @@ public RowResolver getRowResolver(Operator opt) { * the semantic analyzer need to deal with the select statement with respect * to the SerDe and Storage Format. */ - private ASTNode analyzeCreateTable(ASTNode ast, QB qb) - throws SemanticException { + private ASTNode analyzeCreateTable( + ASTNode ast, QB qb, PreCboCtx cboCtx) throws SemanticException { String[] qualifiedTabName = getQualifiedTableName((ASTNode) ast.getChild(0)); String dbDotTab = getDotName(qualifiedTabName); @@ -10626,6 +10892,9 @@ private ASTNode analyzeCreateTable(ASTNode ast, QB qb) throw new SemanticException(ErrorMsg.CTAS_EXTTBL_COEXISTENCE.getMsg()); } command_type = CTAS; + if (cboCtx != null) { + cboCtx.set(PreCboCtx.Type.CTAS, child); + } selectStmt = child; break; case HiveParser.TOK_TABCOLLIST: @@ -12197,27 +12466,43 @@ protected boolean deleting() { /**** Temporary Place Holder For Optiq plan Gen, Optimizer ****/ - /* - * Entry point to Optimizations using Optiq. + /** + * Entry point to Optimizations using Optiq. Checks whether Optiq can handle the query. + * @param qbToChk Query block to check. + * @param verbose Whether return value should be verbose in case of failure. + * @return null if the query can be handled; non-null reason string if it cannot be. */ - private boolean canHandleQuery(QB qbToChk, boolean topLevelQB) { - boolean runOptiqPlanner = false; + private String canHandleQbForCbo(QB qbToChk, boolean topLevelQB, boolean verbose) { // Assumption: // 1. If top level QB is query then everything below it must also be Query // 2. Nested Subquery will return false for qbToChk.getIsQuery() - if ((!topLevelQB || qbToChk.getIsQuery()) - && (!conf.getBoolVar(ConfVars.HIVE_IN_TEST) || conf.getVar(ConfVars.HIVEMAPREDMODE).equalsIgnoreCase("nonstrict")) - && (!topLevelQB || (queryProperties.getJoinCount() > 1) || conf.getBoolVar(ConfVars.HIVE_IN_TEST)) - && !queryProperties.hasClusterBy() && !queryProperties.hasDistributeBy() - && !queryProperties.hasSortBy() && !queryProperties.hasPTF() - && !queryProperties.usesScript() && !queryProperties.hasMultiDestQuery() - && !queryProperties.hasLateralViews()) { - runOptiqPlanner = true; - } else { - LOG.info("Can not invoke CBO; query contains operators not supported for CBO."); - } - - return runOptiqPlanner; + boolean isInTest = conf.getBoolVar(ConfVars.HIVE_IN_TEST); + boolean isStrictTest = isInTest + && !conf.getVar(ConfVars.HIVEMAPREDMODE).equalsIgnoreCase("nonstrict"); + boolean hasEnoughJoins = !topLevelQB || (queryProperties.getJoinCount() > 1) || isInTest; + if (!isStrictTest && hasEnoughJoins && !queryProperties.hasClusterBy() + && !queryProperties.hasDistributeBy() && !queryProperties.hasSortBy() + && !queryProperties.hasPTF() && !queryProperties.usesScript() + && !queryProperties.hasMultiDestQuery() && !queryProperties.hasLateralViews()) { + return null; // Ok to run CBO. + } + + // Not ok to run CBO, build error message. + String msg = ""; + if (verbose) { + if (isStrictTest) msg += "is in test running in mode other than nonstrict; "; + if (!hasEnoughJoins) msg += "has too few joins; "; + if (queryProperties.hasClusterBy()) msg += "has cluster by; "; + if (queryProperties.hasDistributeBy()) msg += "has distribute by; "; + if (queryProperties.hasSortBy()) msg += "has sort by; "; + if (queryProperties.hasPTF()) msg += "has PTF; "; + if (queryProperties.usesScript()) msg += "uses scripts; "; + if (queryProperties.hasMultiDestQuery()) msg += "is a multi-destination query; "; + if (queryProperties.hasLateralViews()) msg += "has lateral views; "; + + if (msg.isEmpty()) msg += "has some unspecified limitations; "; + } + return msg; } private class OptiqBasedPlanner implements Frameworks.PlannerAction { @@ -12225,7 +12510,7 @@ private boolean canHandleQuery(QB qbToChk, boolean topLevelQB) { private RelOptSchema relOptSchema; private SemanticException semanticException; private Map partitionCache; - private AtomicInteger noColsMissingStats = new AtomicInteger(0); + private final AtomicInteger noColsMissingStats = new AtomicInteger(0); List topLevelFieldSchema; // TODO: Do we need to keep track of RR, ColNameToPosMap for every op or @@ -12243,16 +12528,50 @@ private ASTNode getOptimizedAST(Map partitionCache) optimizedOptiqPlan = Frameworks.withPlanner(this, Frameworks.newConfigBuilder().typeSystem(new HiveTypeSystemImpl()).build()); } catch (Exception e) { - if (semanticException != null) - throw semanticException; - else - throw new RuntimeException(e); + rethrowCalciteException(e); + throw new AssertionError("rethrowCalciteException didn't throw for " + e.getMessage()); } optiqOptimizedAST = ASTConverter.convert(optimizedOptiqPlan, topLevelFieldSchema); return optiqOptimizedAST; } + /* + * Unwraps a chain of useless UndeclaredThrowableException-s, InvocationTargetException-s + * and RuntimeException-s potentially coming from CBO/Calcite code. + */ + private void rethrowCalciteException(Exception e) throws SemanticException { + Throwable first = (semanticException != null) ? semanticException : e, + current = first, cause = current.getCause(); + while (cause != null) { + Throwable causeOfCause = cause.getCause(); + if (current == first && causeOfCause == null && isUselessCause(first)) { + // "cause" is a root cause, and "e"/"first" is a useless exception it's wrapped in. + first = cause; + break; + } else if (causeOfCause != null && isUselessCause(cause) + && ExceptionHelper.resetCause(current, causeOfCause)) { + // "cause" was a useless intermediate cause and was replace it with its own cause. + cause = causeOfCause; + continue; // do loop once again with the new cause of "current" + } + current = cause; + cause = current.getCause(); + } + + if (first instanceof RuntimeException) { + throw (RuntimeException)first; + } else if (first instanceof SemanticException) { + throw (SemanticException)first; + } + throw new RuntimeException(first); + } + + private boolean isUselessCause(Throwable t) { + return t instanceof RuntimeException || t instanceof InvocationTargetException + || t instanceof UndeclaredThrowableException; + } + @Override public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlus rootSchema) { RelNode optiqGenPlan = null; @@ -12308,12 +12627,11 @@ public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlu if (LOG.isDebugEnabled() && !conf.getBoolVar(ConfVars.HIVE_IN_TEST)) { LOG.debug("CBO Planning details:\n"); - LOG.debug("Original Plan:\n"); - LOG.debug(RelOptUtil.toString(optiqGenPlan)); - LOG.debug("Plan After PPD, PartPruning, ColumnPruning:\n"); - LOG.debug(RelOptUtil.toString(optiqPreCboPlan)); - LOG.debug("Plan After Join Reordering:\n"); - LOG.debug(RelOptUtil.toString(optiqOptimizedPlan, SqlExplainLevel.ALL_ATTRIBUTES)); + LOG.debug("Original Plan:\n" + RelOptUtil.toString(optiqGenPlan)); + LOG.debug("Plan After PPD, PartPruning, ColumnPruning:\n" + + RelOptUtil.toString(optiqPreCboPlan)); + LOG.debug("Plan After Join Reordering:\n" + + RelOptUtil.toString(optiqOptimizedPlan, SqlExplainLevel.ALL_ATTRIBUTES)); } return optiqOptimizedPlan; @@ -12417,7 +12735,7 @@ private RelNode genUnionLogicalPlan(String unionalias, String leftalias, RelNode + " does not have the field " + field)); } if (!lInfo.getInternalName().equals(rInfo.getInternalName())) { - throw new SemanticException(generateErrorMessage(tabref, + throw new OptiqSemanticException(generateErrorMessage(tabref, "Schema of both sides of union should match: field " + field + ":" + " appears on the left side of the UNION at column position: " + getPositionFromInternalName(lInfo.getInternalName()) @@ -12429,7 +12747,7 @@ private RelNode genUnionLogicalPlan(String unionalias, String leftalias, RelNode TypeInfo commonTypeInfo = FunctionRegistry.getCommonClassForUnionAll(lInfo.getType(), rInfo.getType()); if (commonTypeInfo == null) { - throw new SemanticException(generateErrorMessage(tabref, + throw new OptiqSemanticException(generateErrorMessage(tabref, "Schema of both sides of union should match: Column " + field + " is of type " + lInfo.getType().getTypeName() + " on first table and type " + rInfo.getType().getTypeName() + " on second table")); @@ -12527,7 +12845,9 @@ private RelNode genJoinRelNode(RelNode leftRel, RelNode rightRel, JoinType hiveJ joinRR = RowResolver.getCombinedRR(leftRR, rightRR); } else { joinRR = new RowResolver(); - RowResolver.add(joinRR, leftRR, 0); + if (!RowResolver.add(joinRR, leftRR)) { + LOG.warn("Duplicates detected when adding columns to RR: see previous message"); + } } // 2. Construct ExpressionNodeDesc representing Join Condition @@ -12803,12 +13123,21 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc private RelNode genFilterRelNode(ASTNode filterExpr, RelNode srcRel) throws SemanticException { ExprNodeDesc filterCondn = genExprNodeDesc(filterExpr, relToHiveRR.get(srcRel)); + if (filterCondn instanceof ExprNodeConstantDesc && + !filterCondn.getTypeString().equals(serdeConstants.BOOLEAN_TYPE_NAME)) { + // queries like select * from t1 where 'foo'; + // Optiq's rule PushFilterThroughProject chokes on it. Arguably, we can insert a cast to + // boolean in such cases, but since Postgres, Oracle and MS SQL server fail on compile time + // for such queries, its an arcane corner case, not worth of adding that complexity. + throw new OptiqSemanticException("Filter expression with non-boolean return type."); + } ImmutableMap hiveColNameOptiqPosMap = this.relToHiveColNameOptiqPosMap .get(srcRel); RexNode convertedFilterExpr = new RexNodeConverter(cluster, srcRel.getRowType(), hiveColNameOptiqPosMap, 0, true).convert(filterCondn); + RexNode factoredFilterExpr = RexUtil.pullFactors(cluster.getRexBuilder(), convertedFilterExpr); RelNode filterRel = new HiveFilterRel(cluster, cluster.traitSetOf(HiveRel.CONVENTION), - srcRel, convertedFilterExpr); + srcRel, factoredFilterExpr); this.relToHiveColNameOptiqPosMap.put(filterRel, hiveColNameOptiqPosMap); relToHiveRR.put(filterRel, relToHiveRR.get(srcRel)); relToHiveColNameOptiqPosMap.put(filterRel, hiveColNameOptiqPosMap); @@ -12890,7 +13219,7 @@ private RelNode genFilterRelNode(QB qb, ASTNode searchCond, RelNode srcRel, QB qbSQ = new QB(subQuery.getOuterQueryId(), subQuery.getAlias(), true); qbSQ.setSubQueryDef(subQuery.getSubQuery()); Phase1Ctx ctx_1 = initPhase1Ctx(); - doPhase1(subQuery.getSubQueryAST(), qbSQ, ctx_1); + doPhase1(subQuery.getSubQueryAST(), qbSQ, ctx_1, null); getMetaData(qbSQ); RelNode subQueryRelNode = genLogicalPlan(qbSQ, false); aliasToRel.put(subQuery.getAlias(), subQueryRelNode); @@ -12918,7 +13247,7 @@ private RelNode genFilterRelNode(QB qb, ASTNode searchCond, RelNode srcRel, QB qbSQ_nic = new QB(subQuery.getOuterQueryId(), notInCheck.getAlias(), true); qbSQ_nic.setSubQueryDef(notInCheck.getSubQuery()); ctx_1 = initPhase1Ctx(); - doPhase1(notInCheck.getSubQueryAST(), qbSQ_nic, ctx_1); + doPhase1(notInCheck.getSubQueryAST(), qbSQ_nic, ctx_1, null); getMetaData(qbSQ_nic); RelNode subQueryNICRelNode = genLogicalPlan(qbSQ_nic, false); aliasToRel.put(notInCheck.getAlias(), subQueryNICRelNode); @@ -12962,7 +13291,7 @@ private RelNode genFilterRelNode(QB qb, ASTNode searchCond, RelNode srcRel, private RelNode projectLeftOuterSide(RelNode srcRel, int numColumns) throws SemanticException { RowResolver iRR = relToHiveRR.get(srcRel); RowResolver oRR = new RowResolver(); - RowResolver.add(oRR, iRR, 0, numColumns); + RowResolver.add(oRR, iRR, numColumns); List optiqColLst = new ArrayList(); List oFieldNames = new ArrayList(); @@ -13279,7 +13608,7 @@ private RelNode genGBLogicalPlan(QB qb, RelNode srcRel) throws SemanticException grpbyExpr, new TypeCheckCtx(groupByInputRowResolver)); ExprNodeDesc grpbyExprNDesc = astToExprNDescMap.get(grpbyExpr); if (grpbyExprNDesc == null) - throw new RuntimeException("Invalid Column Reference: " + grpbyExpr.dump()); + throw new OptiqSemanticException("Invalid Column Reference: " + grpbyExpr.dump()); addToGBExpr(groupByOutputRowResolver, groupByInputRowResolver, grpbyExpr, grpbyExprNDesc, gbExprNDescLst, outputColumnNames); @@ -13422,7 +13751,10 @@ public RexNode apply(RelDataTypeField input) { } }); RowResolver obSyntheticProjectRR = new RowResolver(); - RowResolver.add(obSyntheticProjectRR, inputRR, 0); + if (!RowResolver.add(obSyntheticProjectRR, inputRR)) { + throw new OptiqSemanticException( + "Duplicates detected when adding columns to RR: see previous message"); + } int vcolPos = inputRR.getRowSchema().getSignature().size(); for (Pair astTypePair : vcASTTypePairs) { obSyntheticProjectRR.putExpression(astTypePair.getKey(), new ColumnInfo( @@ -13433,14 +13765,23 @@ public RexNode apply(RelDataTypeField input) { obSyntheticProjectRR, srcRel); if (outermostOB) { - RowResolver.add(outputRR, inputRR, 0); + if (!RowResolver.add(outputRR, inputRR)) { + throw new OptiqSemanticException( + "Duplicates detected when adding columns to RR: see previous message"); + } } else { - RowResolver.add(outputRR, obSyntheticProjectRR, 0); + if (!RowResolver.add(outputRR, obSyntheticProjectRR)) { + throw new OptiqSemanticException( + "Duplicates detected when adding columns to RR: see previous message"); + } originalOBChild = srcRel; } } else { - RowResolver.add(outputRR, inputRR, 0); + if (!RowResolver.add(outputRR, inputRR)) { + throw new OptiqSemanticException( + "Duplicates detected when adding columns to RR: see previous message"); + } } // 4. Construct SortRel @@ -13475,7 +13816,10 @@ private RelNode genLimitLogicalPlan(QB qb, RelNode srcRel) throws SemanticExcept sortRel = new HiveSortRel(cluster, traitSet, srcRel, canonizedCollation, null, fetch); RowResolver outputRR = new RowResolver(); - RowResolver.add(outputRR, relToHiveRR.get(srcRel), 0); + if (!RowResolver.add(outputRR, relToHiveRR.get(srcRel))) { + throw new OptiqSemanticException( + "Duplicates detected when adding columns to RR: see previous message"); + } ImmutableMap hiveColNameOptiqPosMap = buildHiveToOptiqColumnMap(outputRR, sortRel); relToHiveRR.put(sortRel, outputRR); @@ -13637,51 +13981,48 @@ int getWindowSpecIndx(ASTNode wndAST) { return new Pair(w, wHiveRetType); } - private RelNode genSelectForWindowing(QB qb, RelNode srcRel) throws SemanticException { - RelNode selOpForWindow = null; + private RelNode genSelectForWindowing( + QB qb, RelNode srcRel, HashSet newColumns) throws SemanticException { QBParseInfo qbp = getQBParseInfo(qb); WindowingSpec wSpec = (!qb.getAllWindowingSpecs().isEmpty()) ? qb.getAllWindowingSpecs() .values().iterator().next() : null; + if (wSpec == null) return null; + // 1. Get valid Window Function Spec + wSpec.validateAndMakeEffective(); + List windowExpressions = wSpec.getWindowExpressions(); + if (windowExpressions == null || windowExpressions.isEmpty()) return null; - if (wSpec != null) { - // 1. Get valid Window Function Spec - wSpec.validateAndMakeEffective(); - List windowExpressions = wSpec.getWindowExpressions(); - - if (windowExpressions != null && !windowExpressions.isEmpty()) { - RowResolver inputRR = this.relToHiveRR.get(srcRel); - // 2. Get RexNodes for original Projections from below - List projsForWindowSelOp = new ArrayList( - HiveOptiqUtil.getProjsFromBelowAsInputRef(srcRel)); - - // 3. Construct new Row Resolver with everything from below. - RowResolver out_rwsch = new RowResolver(); - RowResolver.add(out_rwsch, inputRR, 0); - - // 4. Walk through Window Expressions & Construct RexNodes for those, - // Update out_rwsch - for (WindowExpressionSpec wExprSpec : windowExpressions) { - if (out_rwsch.getExpression(wExprSpec.getExpression()) == null) { - Pair wtp = genWindowingProj(qb, wExprSpec, srcRel); - projsForWindowSelOp.add(wtp.getKey()); - - // 6.2.2 Update Output Row Schema - ColumnInfo oColInfo = new ColumnInfo( - getColumnInternalName(projsForWindowSelOp.size()), wtp.getValue(), null, false); - if (false) { - out_rwsch.checkColumn(null, wExprSpec.getAlias()); - out_rwsch.put(null, wExprSpec.getAlias(), oColInfo); - } else { - out_rwsch.putExpression(wExprSpec.getExpression(), oColInfo); - } - } - } + RowResolver inputRR = this.relToHiveRR.get(srcRel); + // 2. Get RexNodes for original Projections from below + List projsForWindowSelOp = new ArrayList( + HiveOptiqUtil.getProjsFromBelowAsInputRef(srcRel)); - selOpForWindow = genSelectRelNode(projsForWindowSelOp, out_rwsch, srcRel); + // 3. Construct new Row Resolver with everything from below. + RowResolver out_rwsch = new RowResolver(); + if (!RowResolver.add(out_rwsch, inputRR)) { + LOG.warn("Duplicates detected when adding columns to RR: see previous message"); + } + + // 4. Walk through Window Expressions & Construct RexNodes for those, + // Update out_rwsch + for (WindowExpressionSpec wExprSpec : windowExpressions) { + if (out_rwsch.getExpression(wExprSpec.getExpression()) == null) { + Pair wtp = genWindowingProj(qb, wExprSpec, srcRel); + projsForWindowSelOp.add(wtp.getKey()); + + // 6.2.2 Update Output Row Schema + ColumnInfo oColInfo = new ColumnInfo( + getColumnInternalName(projsForWindowSelOp.size()), wtp.getValue(), null, false); + if (false) { + out_rwsch.put(null, wExprSpec.getAlias(), oColInfo); + } else { + out_rwsch.putExpression(wExprSpec.getExpression(), oColInfo); + } + newColumns.add(oColInfo); } } - return selOpForWindow; + return genSelectRelNode(projsForWindowSelOp, out_rwsch, srcRel); } private RelNode genSelectRelNode(List optiqColLst, RowResolver out_rwsch, @@ -13770,9 +14111,10 @@ public String apply(String hName) { * @throws SemanticException */ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel) throws SemanticException { - // 0. Generate a Select Node for Windowing - RelNode selForWindow = genSelectForWindowing(qb, srcRel); + // Exclude the newly-generated select columns from */etc. resolution. + HashSet excludedColumns = new HashSet(); + RelNode selForWindow = genSelectForWindowing(qb, srcRel, excludedColumns); srcRel = (selForWindow == null) ? srcRel : selForWindow; boolean subQuery; @@ -13806,9 +14148,6 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel) throws SemanticExcep throw new OptiqSemanticException(msg); } - // 4. Determine if select corresponds to a subquery - subQuery = qb.getParseInfo().getIsSubQ(); - // 4. Bailout if select involves Transform boolean isInTransform = (selExprList.getChild(posn).getChild(0).getType() == HiveParser.TOK_TRANSFORM); if (isInTransform) { @@ -13819,16 +14158,12 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel) throws SemanticExcep } // 5. Bailout if select involves UDTF - ASTNode udtfExpr = (ASTNode) selExprList.getChild(posn).getChild(0); - GenericUDTF genericUDTF = null; - int udtfExprType = udtfExpr.getType(); - if (udtfExprType == HiveParser.TOK_FUNCTION || udtfExprType == HiveParser.TOK_FUNCTIONSTAR) { - String funcName = TypeCheckProcFactory.DefaultExprProcessor.getFunctionText(udtfExpr, true); + ASTNode expr = (ASTNode) selExprList.getChild(posn).getChild(0); + int exprType = expr.getType(); + if (exprType == HiveParser.TOK_FUNCTION || exprType == HiveParser.TOK_FUNCTIONSTAR) { + String funcName = TypeCheckProcFactory.DefaultExprProcessor.getFunctionText(expr, true); FunctionInfo fi = FunctionRegistry.getFunctionInfo(funcName); - if (fi != null) { - genericUDTF = fi.getGenericUDTF(); - } - if (genericUDTF != null) { + if (fi != null && fi.getGenericUDTF() != null) { String msg = String.format("UDTF " + funcName + " is currently not supported in CBO," + " turn off cbo to use UDTF " + funcName); LOG.debug(msg); @@ -13856,7 +14191,6 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel) throws SemanticExcep ErrorMsg.INVALID_AS.getMsg())); } - ASTNode expr; String tabAlias; String colAlias; @@ -13871,8 +14205,8 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel) throws SemanticExcep if (expr.getType() == HiveParser.TOK_ALLCOLREF) { pos = genColListRegex(".*", expr.getChildCount() == 0 ? null : getUnescapedName((ASTNode) expr.getChild(0)) - .toLowerCase(), expr, col_list, inputRR, pos, out_rwsch, tabAliasesForAllProjs, - subQuery); + .toLowerCase(), expr, col_list, excludedColumns, inputRR, pos, out_rwsch, + tabAliasesForAllProjs, true); selectStar = true; } else if (expr.getType() == HiveParser.TOK_TABLE_OR_COL && !hasAsClause && !inputRR.getIsExprResolver() @@ -13881,7 +14215,7 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel) throws SemanticExcep // This can only happen without AS clause // We don't allow this for ExprResolver - the Group By case pos = genColListRegex(unescapeIdentifier(expr.getChild(0).getText()), null, expr, - col_list, inputRR, pos, out_rwsch, tabAliasesForAllProjs, subQuery); + col_list, excludedColumns, inputRR, pos, out_rwsch, tabAliasesForAllProjs, true); } else if (expr.getType() == HiveParser.DOT && expr.getChild(0).getType() == HiveParser.TOK_TABLE_OR_COL && inputRR.hasTableAlias(unescapeIdentifier(expr.getChild(0).getChild(0).getText() @@ -13892,7 +14226,10 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel) throws SemanticExcep // We don't allow this for ExprResolver - the Group By case pos = genColListRegex(unescapeIdentifier(expr.getChild(1).getText()), unescapeIdentifier(expr.getChild(0).getChild(0).getText().toLowerCase()), expr, - col_list, inputRR, pos, out_rwsch, tabAliasesForAllProjs, subQuery); + col_list, excludedColumns, inputRR, pos, out_rwsch, tabAliasesForAllProjs, true); + } else if (expr.toStringTree().contains("TOK_FUNCTIONDI") && !(srcRel instanceof HiveAggregateRel)) { + // Likely a malformed query eg, select hash(distinct c1) from t1; + throw new OptiqSemanticException("Distinct without an aggreggation."); } else { // Case when this is an expression TypeCheckCtx tcCtx = new TypeCheckCtx(inputRR); @@ -13904,15 +14241,15 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel) throws SemanticExcep colAlias = recommended; } col_list.add(exp); - if (subQuery) { - out_rwsch.checkColumn(tabAlias, colAlias); - } ColumnInfo colInfo = new ColumnInfo(getColumnInternalName(pos), exp.getWritableObjectInspector(), tabAlias, false); colInfo.setSkewedCol((exp instanceof ExprNodeColumnDesc) ? ((ExprNodeColumnDesc) exp) .isSkewedCol() : false); - out_rwsch.put(tabAlias, colAlias, colInfo); + if (!out_rwsch.putWithCheck(tabAlias, colAlias, null, colInfo)) { + throw new OptiqSemanticException("Cannot add column to RR: " + tabAlias + "." + + colAlias + " => " + colInfo + " due to duplication, see previous warnings"); + } if (exp instanceof ExprNodeColumnDesc) { ExprNodeColumnDesc colExp = (ExprNodeColumnDesc) exp; @@ -13969,11 +14306,14 @@ private RelNode genLogicalPlan(QB qb, boolean outerMostQB) throws SemanticExcept // First generate all the opInfos for the elements in the from clause Map aliasToRel = new HashMap(); - // 0. Check if we can handle the query - // This check is needed here because of SubQuery - if (!canHandleQuery(qb, false)) { - String msg = String.format("CBO Can not handle Sub Query"); - LOG.debug(msg); + // 0. Check if we can handle the SubQuery; + // canHandleQbForCbo returns null if the query can be handled. + String reason = canHandleQbForCbo(qb, false, LOG.isDebugEnabled()); + if (reason != null) { + String msg = "CBO can not handle Sub Query"; + if (LOG.isDebugEnabled()) { + LOG.debug(msg + " because it: " + reason); + } throw new OptiqSemanticException(msg); } @@ -13982,7 +14322,6 @@ private RelNode genLogicalPlan(QB qb, boolean outerMostQB) throws SemanticExcept for (String subqAlias : qb.getSubqAliases()) { QBExpr qbexpr = qb.getSubqForAlias(subqAlias); aliasToRel.put(subqAlias, genLogicalPlan(qbexpr)); - qbexpr.setAlias(subqAlias); } // 1.2 Recurse over all the source tables @@ -14065,7 +14404,10 @@ public RexNode apply(RelDataTypeField input) { } }); RowResolver topConstrainingProjRR = new RowResolver(); - RowResolver.add(topConstrainingProjRR, this.relToHiveRR.get(topConstrainingProjArgsRel), 0); + if (!RowResolver.add( + topConstrainingProjRR, this.relToHiveRR.get(topConstrainingProjArgsRel))) { + LOG.warn("Duplicates detected when adding columns to RR: see previous message"); + } srcRel = genSelectRelNode(originalInputRefs, topConstrainingProjRR, srcRel); } @@ -14105,6 +14447,10 @@ private RelNode genGBHavingLogicalPlan(QB qb, RelNode srcRel, Map 0; + throw new OptiqSemanticException("Having clause without any group-by."); + } validateNoHavingReferenceToAlias(qb, (ASTNode) havingClause.getChild(0)); gbFilter = genFilterRelNode(qb, (ASTNode) havingClause.getChild(0), srcRel, aliasToRel, true); @@ -14215,7 +14561,7 @@ private QBParseInfo getQBParseInfo(QB qb) throws OptiqSemanticException { } private List getTabAliases(RowResolver inputRR) { - List tabAliases = new ArrayList(); + List tabAliases = new ArrayList(); // TODO: this should be unique for (ColumnInfo ci : inputRR.getColumnInfos()) { tabAliases.add(ci.getTabAlias()); } @@ -14223,4 +14569,36 @@ private QBParseInfo getQBParseInfo(QB qb) throws OptiqSemanticException { return tabAliases; } } + + private static class ExceptionHelper { + private static final Field CAUSE_FIELD = getField(Throwable.class, "cause"), + TARGET_FIELD = getField(InvocationTargetException.class, "target"), + MESSAGE_FIELD = getField(Throwable.class, "detailMessage"); + private static Field getField(Class clazz, String name) { + try { + Field f = clazz.getDeclaredField(name); + f.setAccessible(true); + return f; + } catch (Throwable t) { + return null; + } + } + public static boolean resetCause(Throwable target, Throwable newCause) { + try { + if (MESSAGE_FIELD == null) return false; + Field field = (target instanceof InvocationTargetException) ? TARGET_FIELD : CAUSE_FIELD; + if (field == null) return false; + + Throwable oldCause = target.getCause(); + String oldMsg = target.getMessage(); + field.set(target, newCause); + if (oldMsg != null && oldMsg.equals(oldCause.toString())) { + MESSAGE_FIELD.set(target, newCause == null ? null : newCause.toString()); + } + } catch (Throwable se) { + return false; + } + return true; + } + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java index 3ef5189..364cad9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java @@ -306,7 +306,9 @@ private void runDynamicPartitionPruning(OptimizeTezProcContext procCtx, Set alterTableTypesWithPartialSpec = + new HashSet(); + + static { + alterTableTypesWithPartialSpec.add(AlterTableDesc.AlterTableTypes.ALTERPROTECTMODE); + alterTableTypesWithPartialSpec.add(AlterTableDesc.AlterTableTypes.ADDCOLS); + alterTableTypesWithPartialSpec.add(AlterTableDesc.AlterTableTypes.REPLACECOLS); + alterTableTypesWithPartialSpec.add(AlterTableDesc.AlterTableTypes.RENAMECOLUMN); + alterTableTypesWithPartialSpec.add(AlterTableDesc.AlterTableTypes.ADDPROPS); + alterTableTypesWithPartialSpec.add(AlterTableDesc.AlterTableTypes.DROPPROPS); + alterTableTypesWithPartialSpec.add(AlterTableDesc.AlterTableTypes.ADDSERDE); + alterTableTypesWithPartialSpec.add(AlterTableDesc.AlterTableTypes.ADDSERDEPROPS); + alterTableTypesWithPartialSpec.add(AlterTableDesc.AlterTableTypes.ADDFILEFORMAT); + } AlterTableTypes op; String oldName; @@ -704,5 +720,8 @@ public boolean getIsDropIfExists() { return isDropIfExists; } + public static boolean doesAlterTableTypeSupportPartialPartitionSpec(AlterTableTypes type) { + return alterTableTypesWithPartialSpec.contains(type); + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java index 83b7f87..8c74465 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.plan; +import java.util.ArrayList; import java.util.LinkedList; import java.util.LinkedHashSet; import java.util.List; @@ -42,6 +43,7 @@ // schema info. List dummyOps; int tag; + private final List sortColNames = new ArrayList(); public BaseWork() {} @@ -54,8 +56,8 @@ public BaseWork(String name) { private String name; // Vectorization. - protected Map> scratchColumnVectorTypes = null; - protected Map> scratchColumnMap = null; + protected Map> allScratchColumnVectorTypeMaps = null; + protected Map> allColumnVectorMaps = null; protected boolean vectorMode = false; public void setGatheringStats(boolean gatherStats) { @@ -138,21 +140,21 @@ public void addDummyOp(HashTableDummyOperator dummyOp) { return returnSet; } - public Map> getScratchColumnVectorTypes() { - return scratchColumnVectorTypes; + public Map> getAllScratchColumnVectorTypeMaps() { + return allScratchColumnVectorTypeMaps; } - public void setScratchColumnVectorTypes( - Map> scratchColumnVectorTypes) { - this.scratchColumnVectorTypes = scratchColumnVectorTypes; + public void setAllScratchColumnVectorTypeMaps( + Map> allScratchColumnVectorTypeMaps) { + this.allScratchColumnVectorTypeMaps = allScratchColumnVectorTypeMaps; } - public Map> getScratchColumnMap() { - return scratchColumnMap; + public Map> getAllColumnVectorMaps() { + return allColumnVectorMaps; } - public void setScratchColumnMap(Map> scratchColumnMap) { - this.scratchColumnMap = scratchColumnMap; + public void setAllColumnVectorMaps(Map> allColumnVectorMaps) { + this.allColumnVectorMaps = allColumnVectorMaps; } @Override @@ -173,4 +175,12 @@ public void setTag(int tag) { public int getTag() { return tag; } + + public void addSortCols(List sortCols) { + this.sortColNames.addAll(sortCols); + } + + public List getSortCols() { + return sortColNames; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ColStatistics.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ColStatistics.java index d1b7dea..c420190 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ColStatistics.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ColStatistics.java @@ -33,12 +33,14 @@ private long numTrues; private long numFalses; private Range range; + private boolean isPrimaryKey; public ColStatistics(String tabAlias, String colName, String colType) { this.setTableAlias(tabAlias); this.setColumnName(colName); this.setColumnType(colType); this.setFullyQualifiedColName(StatsUtils.getFullyQualifiedColumnName(tabAlias, colName)); + this.setPrimaryKey(false); } public ColStatistics() { @@ -150,6 +152,12 @@ public String toString() { sb.append(numTrues); sb.append(" numFalses: "); sb.append(numFalses); + if (range != null) { + sb.append(" "); + sb.append(range); + } + sb.append(" isPrimaryKey: "); + sb.append(isPrimaryKey); return sb.toString(); } @@ -162,24 +170,47 @@ public ColStatistics clone() throws CloneNotSupportedException { clone.setNumNulls(numNulls); clone.setNumTrues(numTrues); clone.setNumFalses(numFalses); + clone.setPrimaryKey(isPrimaryKey); if (range != null ) { clone.setRange(range.clone()); } return clone; } + public boolean isPrimaryKey() { + return isPrimaryKey; + } + + public void setPrimaryKey(boolean isPrimaryKey) { + this.isPrimaryKey = isPrimaryKey; + } + public static class Range { public final Number minValue; public final Number maxValue; + Range(Number minValue, Number maxValue) { super(); this.minValue = minValue; this.maxValue = maxValue; } + @Override public Range clone() { return new Range(minValue, maxValue); } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("Range: ["); + sb.append(" min: "); + sb.append(minValue); + sb.append(" max: "); + sb.append(maxValue); + sb.append(" ]"); + return sb.toString(); + } } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/DropIndexDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/DropIndexDesc.java index 5ef14ef..0d5844a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/DropIndexDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/DropIndexDesc.java @@ -24,15 +24,17 @@ private String indexName; private String tableName; + + private boolean throwException; /** * @param indexName * @param tableName */ - public DropIndexDesc(String indexName, String tableName) { - super(); + public DropIndexDesc(String indexName, String tableName, boolean throwException) { this.indexName = indexName; this.tableName = tableName; + this.throwException = throwException; } /** @@ -63,4 +65,11 @@ public void setTableName(String tableName) { this.tableName = tableName; } + public boolean isThrowException() { + return throwException; + } + + public void setThrowException(boolean throwException) { + this.throwException = throwException; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java index 8a41577..b15df0f 100755 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java @@ -40,6 +40,17 @@ // If this constant was created while doing constant folding, foldedFromCol holds the name of // original column from which it was folded. private transient String foldedFromCol; + // string representation of folding constant. + private transient String foldedFromVal; + + public ExprNodeConstantDesc setFoldedFromVal(String foldedFromVal) { + this.foldedFromVal = foldedFromVal; + return this; + } + + public String getFoldedFromVal() { + return foldedFromVal; + } public String getFoldedFromCol() { return foldedFromCol; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDesc.java index 82d86ee..0fe9eda 100755 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDesc.java @@ -19,6 +19,8 @@ package org.apache.hadoop.hive.ql.plan; import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; import java.util.List; import org.apache.hadoop.hive.ql.lib.Node; @@ -125,5 +127,19 @@ public boolean equals(Object other) { public int hashCode() { return exprNodeDesc == null ? 0 : exprNodeDesc.hashCode(); } + + /* helper function to allow Set()/Collection() operations with ExprNodeDesc */ + public static Collection transform( + Collection descs) { + if (descs == null) { + return null; + } + final Collection wrapped = new ArrayList( + descs.size()); + for (ExprNodeDesc desc : descs) { + wrapped.add(new ExprNodeDescEqualityWrapper(desc)); + } + return wrapped; + } } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java index 75372cd..686befd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java @@ -212,7 +212,14 @@ public static ExprNodeDesc backtrack(ExprNodeDesc source, Operator current, if (source instanceof ExprNodeGenericFuncDesc) { // all children expression should be resolved ExprNodeGenericFuncDesc function = (ExprNodeGenericFuncDesc) source.clone(); - function.setChildren(backtrack(function.getChildren(), current, terminal)); + List children = backtrack(function.getChildren(), current, terminal); + for (ExprNodeDesc child : children) { + if (child == null) { + // Could not resolve all of the function children, fail + return null; + } + } + function.setChildren(children); return function; } if (source instanceof ExprNodeColumnDesc) { @@ -222,7 +229,11 @@ public static ExprNodeDesc backtrack(ExprNodeDesc source, Operator current, if (source instanceof ExprNodeFieldDesc) { // field expression should be resolved ExprNodeFieldDesc field = (ExprNodeFieldDesc) source.clone(); - field.setDesc(backtrack(field.getDesc(), current, terminal)); + ExprNodeDesc fieldDesc = backtrack(field.getDesc(), current, terminal); + if (fieldDesc == null) { + return null; + } + field.setDesc(fieldDesc); return field; } // constant or null expr, just return diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java index 2c4175a..57beb69 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.plan; import java.util.ArrayList; +import java.util.EnumSet; import java.util.List; import org.apache.commons.logging.Log; @@ -90,7 +91,22 @@ //flag used to control how TopN handled for PTF/Windowing partitions. private boolean isPTFReduceSink = false; private boolean skipTag; // Skip writing tags when feeding into mapjoin hashtable - private Boolean autoParallel = null; // Is reducer auto-parallelism enabled, disabled or unset + + public static enum ReducerTraits { + UNSET(0), // unset + FIXED(1), // distribution of keys is fixed + AUTOPARALLEL(2), // can change reducer count (ORDER BY can concat adjacent buckets) + UNIFORM(3); // can redistribute into buckets uniformly (GROUP BY can) + + private final int trait; + + private ReducerTraits(int trait) { + this.trait = trait; + } + }; + + // Is reducer auto-parallelism unset (FIXED, UNIFORM, PARALLEL) + private EnumSet reduceTraits = EnumSet.of(ReducerTraits.UNSET); // Write type, since this needs to calculate buckets differently for updates and deletes private AcidUtils.Operation writeType; @@ -148,7 +164,7 @@ public Object clone() { desc.setBucketCols(bucketCols); desc.setStatistics(this.getStatistics()); desc.setSkipTag(skipTag); - desc.autoParallel = autoParallel; + desc.reduceTraits = reduceTraits.clone(); return desc; } @@ -361,16 +377,30 @@ public boolean getSkipTag() { @Explain(displayName = "auto parallelism", normalExplain = false) public final boolean isAutoParallel() { - return (autoParallel != null) && autoParallel; + return (this.reduceTraits.contains(ReducerTraits.AUTOPARALLEL)); + } + + public final EnumSet getReducerTraits() { + return this.reduceTraits; } - public final void setAutoParallel(final boolean autoParallel) { + public final void setReducerTraits(EnumSet traits) { // we don't allow turning on auto parallel once it has been // explicitly turned off. That is to avoid scenarios where // auto parallelism could break assumptions about number of // reducers or hash function. - if (this.autoParallel == null || this.autoParallel == true) { - this.autoParallel = autoParallel; + + boolean wasUnset = this.reduceTraits.remove(ReducerTraits.UNSET); + + if (this.reduceTraits.contains(ReducerTraits.FIXED)) { + return; + } else if (traits.contains(ReducerTraits.FIXED)) { + this.reduceTraits.removeAll(EnumSet.of( + ReducerTraits.AUTOPARALLEL, + ReducerTraits.UNIFORM)); + this.reduceTraits.addAll(traits); + } else { + this.reduceTraits.addAll(traits); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java index 53ba031..c78184b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java @@ -95,8 +95,6 @@ public ReduceWork(String name) { private ObjectInspector keyObjectInspector = null; private ObjectInspector valueObjectInspector = null; - private final Map reduceColumnNameMap = new LinkedHashMap(); - /** * If the plan has a reducer and correspondingly a reduce-sink, then store the TableDesc pointing * to keySerializeInfo of the ReduceSink @@ -142,58 +140,6 @@ public ObjectInspector getValueObjectInspector() { return valueObjectInspector; } - private int addToReduceColumnNameMap(StructObjectInspector structObjectInspector, int startIndex, String prefix) { - List fields = structObjectInspector.getAllStructFieldRefs(); - int index = startIndex; - for (StructField field: fields) { - reduceColumnNameMap.put(prefix + "." + field.getFieldName(), index); - index++; - } - return index; - } - - public Boolean fillInReduceColumnNameMap() { - ObjectInspector keyObjectInspector = getKeyObjectInspector(); - if (keyObjectInspector == null || !(keyObjectInspector instanceof StructObjectInspector)) { - return false; - } - StructObjectInspector keyStructObjectInspector = (StructObjectInspector) keyObjectInspector; - - ObjectInspector valueObjectInspector = getValueObjectInspector(); - if (valueObjectInspector == null || !(valueObjectInspector instanceof StructObjectInspector)) { - return false; - } - StructObjectInspector valueStructObjectInspector = (StructObjectInspector) valueObjectInspector; - - int keyCount = addToReduceColumnNameMap(keyStructObjectInspector, 0, Utilities.ReduceField.KEY.toString()); - addToReduceColumnNameMap(valueStructObjectInspector, keyCount, Utilities.ReduceField.VALUE.toString()); - return true; - } - - public Map getReduceColumnNameMap() { - if (needsTagging) { - return null; - } - if (reduceColumnNameMap.size() == 0) { - if (!fillInReduceColumnNameMap()) { - return null; - } - } - return reduceColumnNameMap; - } - - public List getReduceColumnNames() { - if (needsTagging) { - return null; - } - if (reduceColumnNameMap.size() == 0) { - if (!fillInReduceColumnNameMap()) { - return null; - } - } - return new ArrayList(reduceColumnNameMap.keySet()); - } - public List getTagToValueDesc() { return tagToValueDesc; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowFunctionsDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowFunctionsDesc.java index 5ea99f8..5d4a821 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowFunctionsDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowFunctionsDesc.java @@ -32,6 +32,10 @@ String pattern; String resFile; /** + * whether like keyword is specified + */ + private boolean isLikePattern = false; + /** * table name for the result of show tables. */ private static final String table = "show"; @@ -69,6 +73,18 @@ public ShowFunctionsDesc(Path resFile, String pattern) { } /** + * @param pattern + * names of tables to show + * @param like + * is like keyword used + */ + public ShowFunctionsDesc(Path resFile, String pattern, boolean isLikePattern) { + this(resFile, pattern); + this.isLikePattern = isLikePattern; + } + + + /** * @return the pattern */ @Explain(displayName = "pattern") @@ -99,4 +115,11 @@ public String getResFile() { public void setResFile(String resFile) { this.resFile = resFile; } + + /** + * @return isLikePattern + */ + public boolean getIsLikePattern() { + return isLikePattern; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java index 78d4d1f..408ddf5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java @@ -24,7 +24,7 @@ import java.util.Map; import java.util.Properties; -import org.apache.hadoop.hive.common.JavaUtils; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils; @@ -76,12 +76,16 @@ public TableDesc( return inputFileFormatClass; } + public Deserializer getDeserializer() throws Exception { + return getDeserializer(null); + } + /** * Return a deserializer object corresponding to the tableDesc. */ - public Deserializer getDeserializer() throws Exception { + public Deserializer getDeserializer(Configuration conf) throws Exception { Deserializer de = getDeserializerClass().newInstance(); - SerDeUtils.initializeSerDe(de, null, properties, null); + SerDeUtils.initializeSerDe(de, conf, properties, null); return de; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java index c6894b3..ffaf5cc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java @@ -149,17 +149,17 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, } } - + public static class PTFPPD extends ScriptPPD { - + /* * For WindowingTableFunction if: - * a. there is a Rank/DenseRank function: if there are unpushedPred of the form + * a. there is a Rank/DenseRank function: if there are unpushedPred of the form * rnkValue < Constant; then use the smallest Constant val as the 'rankLimit' * on the WindowingTablFn. - * b. If there are no Wdw Fns with an End Boundary past the current row, the + * b. If there are no Wdw Fns with an End Boundary past the current row, the * condition can be pushed down as a limit pushdown(mapGroupBy=true) - * + * * (non-Javadoc) * @see org.apache.hadoop.hive.ql.ppd.OpProcFactory.ScriptPPD#process(org.apache.hadoop.hive.ql.lib.Node, java.util.Stack, org.apache.hadoop.hive.ql.lib.NodeProcessorCtx, java.lang.Object[]) */ @@ -170,30 +170,30 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, + ((Operator) nd).getIdentifier() + ")"); OpWalkerInfo owi = (OpWalkerInfo) procCtx; PTFOperator ptfOp = (PTFOperator) nd; - + pushRankLimit(ptfOp, owi); return super.process(nd, stack, procCtx, nodeOutputs); } - + private void pushRankLimit(PTFOperator ptfOp, OpWalkerInfo owi) throws SemanticException { PTFDesc conf = ptfOp.getConf(); - + if ( !conf.forWindowing() ) { return; } - + float threshold = owi.getParseContext().getConf().getFloatVar(HiveConf.ConfVars.HIVELIMITPUSHDOWNMEMORYUSAGE); if (threshold <= 0 || threshold >= 1) { return; } - + WindowTableFunctionDef wTFn = (WindowTableFunctionDef) conf.getFuncDef(); List rFnIdxs = rankingFunctions(wTFn); - + if ( rFnIdxs.size() == 0 ) { return; } - + ExprWalkerInfo childInfo = getChildWalkerInfo(ptfOp, owi); if (childInfo == null) { @@ -207,7 +207,7 @@ private void pushRankLimit(PTFOperator ptfOp, OpWalkerInfo owi) throws SemanticE preds = ExprNodeDescUtils.split(pred, preds); } } - + int rLimit = -1; int fnIdx = -1; for(ExprNodeDesc pred : preds) { @@ -219,7 +219,7 @@ private void pushRankLimit(PTFOperator ptfOp, OpWalkerInfo owi) throws SemanticE } } } - + if ( rLimit != -1 ) { wTFn.setRankLimit(rLimit); wTFn.setRankLimitFunction(fnIdx); @@ -228,68 +228,68 @@ private void pushRankLimit(PTFOperator ptfOp, OpWalkerInfo owi) throws SemanticE } } } - + private List rankingFunctions(WindowTableFunctionDef wTFn) { List rFns = new ArrayList(); for(int i=0; i < wTFn.getWindowFunctions().size(); i++ ) { WindowFunctionDef wFnDef = wTFn.getWindowFunctions().get(i); - if ( (wFnDef.getWFnEval() instanceof GenericUDAFRankEvaluator) || + if ( (wFnDef.getWFnEval() instanceof GenericUDAFRankEvaluator) || (wFnDef.getWFnEval() instanceof GenericUDAFDenseRankEvaluator ) ) { rFns.add(i); } } return rFns; } - + /* * For a predicate check if it is a candidate for pushing down as limit optimization. * The expression must be of the form rankFn <|<= constant. */ private int[] getLimit(WindowTableFunctionDef wTFn, List rFnIdxs, ExprNodeDesc expr) { - + if ( !(expr instanceof ExprNodeGenericFuncDesc) ) { return null; } - + ExprNodeGenericFuncDesc fExpr = (ExprNodeGenericFuncDesc) expr; - - if ( !(fExpr.getGenericUDF() instanceof GenericUDFOPLessThan) && + + if ( !(fExpr.getGenericUDF() instanceof GenericUDFOPLessThan) && !(fExpr.getGenericUDF() instanceof GenericUDFOPEqualOrLessThan) ) { return null; } - + if ( !(fExpr.getChildren().get(0) instanceof ExprNodeColumnDesc) ) { return null; } - + if ( !(fExpr.getChildren().get(1) instanceof ExprNodeConstantDesc) ) { return null; } - + ExprNodeConstantDesc constantExpr = (ExprNodeConstantDesc) fExpr.getChildren().get(1) ; - + if ( constantExpr.getTypeInfo() != TypeInfoFactory.intTypeInfo ) { return null; } - + int limit = (Integer) constantExpr.getValue(); if ( fExpr.getGenericUDF() instanceof GenericUDFOPEqualOrLessThan ) { limit = limit + 1; } String colName = ((ExprNodeColumnDesc)fExpr.getChildren().get(0)).getColumn(); - + for(int i=0; i < rFnIdxs.size(); i++ ) { String fAlias = wTFn.getWindowFunctions().get(i).getAlias(); if ( fAlias.equals(colName)) { return new int[] {limit,i}; } } - + return null; } - + /* - * Limit can be pushed down to Map-side if all Window Functions need access + * Limit can be pushed down to Map-side if all Window Functions need access * to rows before the current row. This is true for: * 1. Rank, DenseRank and Lead Fns. (the window doesn't matter for lead fn). * 2. If the Window for the function is Row based and the End Boundary doesn't @@ -298,8 +298,8 @@ private void pushRankLimit(PTFOperator ptfOp, OpWalkerInfo owi) throws SemanticE private boolean canPushLimitToReduceSink(WindowTableFunctionDef wTFn) { for(WindowFunctionDef wFnDef : wTFn.getWindowFunctions() ) { - if ( (wFnDef.getWFnEval() instanceof GenericUDAFRankEvaluator) || - (wFnDef.getWFnEval() instanceof GenericUDAFDenseRankEvaluator ) || + if ( (wFnDef.getWFnEval() instanceof GenericUDAFRankEvaluator) || + (wFnDef.getWFnEval() instanceof GenericUDAFDenseRankEvaluator ) || (wFnDef.getWFnEval() instanceof GenericUDAFLeadEvaluator ) ) { continue; } @@ -314,18 +314,18 @@ private boolean canPushLimitToReduceSink(WindowTableFunctionDef wTFn) { } return true; } - + private void pushRankLimitToRedSink(PTFOperator ptfOp, HiveConf conf, int rLimit) throws SemanticException { - + Operator parent = ptfOp.getParentOperators().get(0); Operator gP = parent == null ? null : parent.getParentOperators().get(0); - + if ( gP == null || !(gP instanceof ReduceSinkOperator )) { return; } - + float threshold = conf.getFloatVar(HiveConf.ConfVars.HIVELIMITPUSHDOWNMEMORYUSAGE); - + ReduceSinkOperator rSink = (ReduceSinkOperator) gP; ReduceSinkDesc rDesc = rSink.getConf(); rDesc.setTopN(rLimit); @@ -543,7 +543,7 @@ protected Object handlePredicates(Node nd, ExprWalkerInfo prunePreds, OpWalkerIn private void applyFilterTransitivity(JoinOperator nd, OpWalkerInfo owi) throws SemanticException { ExprWalkerInfo prunePreds = - owi.getPrunedPreds((Operator) nd); + owi.getPrunedPreds(nd); if (prunePreds != null) { // We want to use the row resolvers of the parents of the join op // because the rowresolver refers to the output columns of an operator @@ -579,9 +579,6 @@ private void applyFilterTransitivity(JoinOperator nd, OpWalkerInfo owi) int numColumns = eqExpressions.size(); int numEqualities = eqExpressions.get(0).size(); - // joins[i] is the join between table i and i+1 in the JoinOperator - JoinCondDesc[] joins = (nd).getConf().getConds(); - // oldFilters contains the filters to be pushed down Map> oldFilters = prunePreds.getFinalCandidates(); @@ -632,10 +629,32 @@ private void applyFilterTransitivity(JoinOperator nd, OpWalkerInfo owi) } } } + // Push where false filter transitively + Map> candidates = prunePreds.getNonFinalCandidates(); + List exprs; + // where false is not associated with any alias in candidates + if (null != candidates && candidates.get(null) != null && ((exprs = candidates.get(null)) != null)) { + Iterator itr = exprs.iterator(); + while (itr.hasNext()) { + ExprNodeDesc expr = itr.next(); + if (expr instanceof ExprNodeConstantDesc && Boolean.FALSE.equals(((ExprNodeConstantDesc)expr).getValue())) { + // push this 'where false' expr to all aliases + for (String alias : aliasToRR.keySet()) { + List pushedFilters = newFilters.get(alias); + if (null == pushedFilters) { + newFilters.put(alias, new ArrayList()); + } + newFilters.get(alias).add(expr); + } + // this filter is pushed, we can remove it from non-final candidates. + itr.remove(); + } + } + } for (Entry> aliasToFilters : newFilters.entrySet()){ - owi.getPrunedPreds((Operator) nd) + owi.getPrunedPreds(nd) .addPushDowns(aliasToFilters.getKey(), aliasToFilters.getValue()); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAccessController.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAccessController.java index a18eedc..325fba7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAccessController.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAccessController.java @@ -68,5 +68,5 @@ void revokeRole(List hivePrincipals, List roles, boolean List getRoleGrantInfoForPrincipal(HivePrincipal principal) throws HiveAuthzPluginException, HiveAccessControlException; - void applyAuthorizationConfigPolicy(HiveConf hiveConf); + void applyAuthorizationConfigPolicy(HiveConf hiveConf) throws HiveAuthzPluginException; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizer.java index 9c3a95c..1f1eba2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizer.java @@ -191,8 +191,9 @@ void checkPrivileges(HiveOperationType hiveOpType, List inp * Modify the given HiveConf object to configure authorization related parameters * or other parameters related to hive security * @param hiveConf + * @throws HiveAuthzPluginException */ - public void applyAuthorizationConfigPolicy(HiveConf hiveConf); + public void applyAuthorizationConfigPolicy(HiveConf hiveConf) throws HiveAuthzPluginException; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizerImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizerImpl.java index 69b803a..e615049 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizerImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizerImpl.java @@ -124,7 +124,7 @@ public void setCurrentRole(String roleName) throws HiveAccessControlException, H } @Override - public void applyAuthorizationConfigPolicy(HiveConf hiveConf) { + public void applyAuthorizationConfigPolicy(HiveConf hiveConf) throws HiveAuthzPluginException { accessController.applyAuthorizationConfigPolicy(hiveConf); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/SettableConfigUpdater.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/SettableConfigUpdater.java new file mode 100644 index 0000000..3de9782 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/SettableConfigUpdater.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.security.authorization.plugin; + +import org.apache.hadoop.hive.common.classification.InterfaceAudience.LimitedPrivate; +import org.apache.hadoop.hive.common.classification.InterfaceStability.Evolving; +import org.apache.hadoop.hive.common.classification.InterfaceStability.Unstable; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; + +/** + * Helper class that can be used by authorization implementations to set a + * default list of 'safe' HiveConf parameters that can be edited by user. It + * uses HiveConf white list parameters to enforce this. This can be called from + * HiveAuthorizer.applyAuthorizationConfigPolicy + * + * The set of config parameters that can be set is restricted to parameters that + * don't allow for any code injection, and config parameters that are not + * considered an 'admin config' option. + * + */ +@LimitedPrivate(value = { "Apache Argus (incubating)" }) +@Evolving +@Unstable +public class SettableConfigUpdater { + + public static void setHiveConfWhiteList(HiveConf hiveConf) throws HiveAuthzPluginException { + + String whiteListParamsStr = hiveConf + .getVar(ConfVars.HIVE_AUTHORIZATION_SQL_STD_AUTH_CONFIG_WHITELIST); + + if(whiteListParamsStr == null && whiteListParamsStr.trim().isEmpty()) { + throw new HiveAuthzPluginException("Configuration parameter " + + ConfVars.HIVE_AUTHORIZATION_SQL_STD_AUTH_CONFIG_WHITELIST.varname + + " is not iniatialized."); + } + + // append regexes that user wanted to add + String whiteListAppend = hiveConf + .getVar(ConfVars.HIVE_AUTHORIZATION_SQL_STD_AUTH_CONFIG_WHITELIST_APPEND); + if (whiteListAppend != null && !whiteListAppend.trim().equals("")) { + whiteListParamsStr = whiteListParamsStr + "|" + whiteListAppend; + } + + hiveConf.setModifiableWhiteListRegex(whiteListParamsStr); + } + +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAccessController.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAccessController.java index 6708425..9c78876 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAccessController.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAccessController.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -59,9 +60,9 @@ import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject; import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject.HivePrivilegeObjectType; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveRoleGrant; +import org.apache.hadoop.hive.ql.security.authorization.plugin.SettableConfigUpdater; import org.apache.thrift.TException; -import com.google.common.base.Joiner; import com.google.common.collect.ImmutableSet; /** @@ -350,9 +351,9 @@ public void revokeRole(List hivePrincipals, List roleName @Override public List getPrincipalGrantInfoForRole(String roleName) throws HiveAuthzPluginException, HiveAccessControlException { // only user belonging to admin role can list role - if (!isUserAdmin()) { + if (!isUserAdmin() && !doesUserHasAdminOption(Arrays.asList(roleName))) { throw new HiveAccessControlException("Current user : " + currentUserName+ " is not" - + " allowed get principals in a role. " + ADMIN_ONLY_MSG); + + " allowed get principals in a role. " + ADMIN_ONLY_MSG + " Otherwise, " + HAS_ADMIN_PRIV_MSG); } try { return getHiveRoleGrants(metastoreClientFactory.getHiveMetastoreClient(), roleName); @@ -609,72 +610,8 @@ private boolean doesUserHasAdminOption(List roleNames) throws HiveAuthzP } } - - /** - * Default list of modifiable config parameters for sql standard authorization - */ - static final String [] defaultModWhiteListSqlStdAuth = new String [] { - ConfVars.BYTESPERREDUCER.varname, - ConfVars.MAXREDUCERS.varname, - ConfVars.HIVEMAPSIDEAGGREGATE.varname, - ConfVars.HIVEMAPAGGRHASHMEMORY.varname, - ConfVars.HIVEMAPAGGRMEMORYTHRESHOLD.varname, - ConfVars.HIVEMAPAGGRHASHMINREDUCTION.varname, - ConfVars.HIVEGROUPBYSKEW.varname, - ConfVars.HIVE_OPTIMIZE_MULTI_GROUPBY_COMMON_DISTINCTS.varname, - ConfVars.HIVEOPTGBYUSINGINDEX.varname, - ConfVars.HIVEOPTPPD.varname, - ConfVars.HIVEOPTPPD_STORAGE.varname, - ConfVars.HIVEOPTPPD_STORAGE.varname, - ConfVars.HIVEPPDRECOGNIZETRANSITIVITY.varname, - ConfVars.HIVEOPTGROUPBY.varname, - ConfVars.HIVEOPTSORTDYNAMICPARTITION.varname, - ConfVars.HIVE_OPTIMIZE_SKEWJOIN_COMPILETIME.varname, - ConfVars.HIVE_OPTIMIZE_UNION_REMOVE.varname, - ConfVars.HIVEMULTIGROUPBYSINGLEREDUCER.varname, - ConfVars.HIVE_MAP_GROUPBY_SORT.varname, - ConfVars.HIVE_MAP_GROUPBY_SORT_TESTMODE.varname, - ConfVars.HIVESKEWJOIN.varname, - ConfVars.HIVE_OPTIMIZE_SKEWJOIN_COMPILETIME.varname, - ConfVars.HIVEMAPREDMODE.varname, - ConfVars.HIVEENFORCEBUCKETMAPJOIN.varname, - ConfVars.COMPRESSRESULT.varname, - ConfVars.COMPRESSINTERMEDIATE.varname, - ConfVars.EXECPARALLEL.varname, - ConfVars.EXECPARALLETHREADNUMBER.varname, - ConfVars.EXECPARALLETHREADNUMBER.varname, - ConfVars.HIVEROWOFFSET.varname, - ConfVars.HIVEMERGEMAPFILES.varname, - ConfVars.HIVEMERGEMAPREDFILES.varname, - ConfVars.HIVEMERGETEZFILES.varname, - ConfVars.HIVEIGNOREMAPJOINHINT.varname, - ConfVars.HIVECONVERTJOIN.varname, - ConfVars.HIVECONVERTJOINNOCONDITIONALTASK.varname, - ConfVars.HIVECONVERTJOINNOCONDITIONALTASKTHRESHOLD.varname, - ConfVars.HIVECONVERTJOINUSENONSTAGED.varname, - ConfVars.HIVECONVERTJOINNOCONDITIONALTASK.varname, - ConfVars.HIVECONVERTJOINNOCONDITIONALTASKTHRESHOLD.varname, - ConfVars.HIVECONVERTJOINUSENONSTAGED.varname, - ConfVars.HIVEENFORCEBUCKETING.varname, - ConfVars.HIVEENFORCESORTING.varname, - ConfVars.HIVEENFORCESORTMERGEBUCKETMAPJOIN.varname, - ConfVars.HIVE_AUTO_SORTMERGE_JOIN.varname, - ConfVars.HIVE_EXECUTION_ENGINE.varname, - ConfVars.HIVE_VECTORIZATION_ENABLED.varname, - ConfVars.HIVEMAPJOINUSEOPTIMIZEDKEYS.varname, - ConfVars.HIVEMAPJOINLAZYHASHTABLE.varname, - ConfVars.HIVE_CHECK_CROSS_PRODUCT.varname, - ConfVars.HIVE_COMPAT.varname, - ConfVars.DYNAMICPARTITIONINGMODE.varname, - "mapred.reduce.tasks", - "mapred.output.compression.codec", - "mapred.map.output.compression.codec", - "mapreduce.job.reduce.slowstart.completedmaps", - "mapreduce.job.queuename", - }; - @Override - public void applyAuthorizationConfigPolicy(HiveConf hiveConf) { + public void applyAuthorizationConfigPolicy(HiveConf hiveConf) throws HiveAuthzPluginException { // First apply configuration applicable to both Hive Cli and HiveServer2 // Not adding any authorization related restrictions to hive cli // grant all privileges for table to its owner - set this in cli as well so that owner @@ -682,28 +619,21 @@ public void applyAuthorizationConfigPolicy(HiveConf hiveConf) { hiveConf.setVar(ConfVars.HIVE_AUTHORIZATION_TABLE_OWNER_GRANTS, "INSERT,SELECT,UPDATE,DELETE"); // Apply rest of the configuration only to HiveServer2 - if(sessionCtx.getClientType() == CLIENT_TYPE.HIVESERVER2) { + if (sessionCtx.getClientType() == CLIENT_TYPE.HIVESERVER2 + && hiveConf.getBoolVar(ConfVars.HIVE_AUTHORIZATION_ENABLED)) { + // Configure PREEXECHOOKS with DisallowTransformHook to disallow transform queries String hooks = hiveConf.getVar(ConfVars.PREEXECHOOKS).trim(); if (hooks.isEmpty()) { hooks = DisallowTransformHook.class.getName(); } else { - hooks = hooks + "," +DisallowTransformHook.class.getName(); + hooks = hooks + "," + DisallowTransformHook.class.getName(); } LOG.debug("Configuring hooks : " + hooks); hiveConf.setVar(ConfVars.PREEXECHOOKS, hooks); - // restrict the variables that can be set using set command to a list in whitelist - hiveConf.setIsModWhiteListEnabled(true); - String whiteListParamsStr = hiveConf.getVar(ConfVars.HIVE_AUTHORIZATION_SQL_STD_AUTH_CONFIG_WHITELIST); - if (whiteListParamsStr == null || whiteListParamsStr.trim().equals("")){ - // set the default configs in whitelist - whiteListParamsStr = Joiner.on(",").join(defaultModWhiteListSqlStdAuth); - hiveConf.setVar(ConfVars.HIVE_AUTHORIZATION_SQL_STD_AUTH_CONFIG_WHITELIST, whiteListParamsStr); - } - for(String whiteListParam : whiteListParamsStr.split(",")){ - hiveConf.addToModifiableWhiteList(whiteListParam); - } + SettableConfigUpdater.setHiveConfWhiteList(hiveConf); + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAccessControllerWrapper.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAccessControllerWrapper.java index 29c1034..a7f47ce 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAccessControllerWrapper.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAccessControllerWrapper.java @@ -174,7 +174,7 @@ public void setCurrentRole(String roleName) throws HiveAuthzPluginException, } @Override - public void applyAuthorizationConfigPolicy(HiveConf hiveConf) { + public void applyAuthorizationConfigPolicy(HiveConf hiveConf) throws HiveAuthzPluginException { hiveAccessController.applyAuthorizationConfigPolicy(hiveConf); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java index 5143fbe..1b27c31 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java @@ -29,6 +29,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.StatsSetupConst; @@ -86,6 +87,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableStringObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableTimestampObjectInspector; import org.apache.hadoop.io.BytesWritable; +import org.apache.tez.mapreduce.hadoop.MRJobConfig; import com.google.common.base.Joiner; import com.google.common.collect.Lists; @@ -175,6 +177,9 @@ public static Statistics collectStatistics(HiveConf conf, PrunedPartitionList pa colStats = getTableColumnStats(table, schema, neededColumns); } + // infer if any column can be primary key based on column statistics + inferAndSetPrimaryKey(stats.getNumRows(), colStats); + stats.setColumnStatsState(deriveStatType(colStats, neededColumns)); stats.addToColumnStats(colStats); } else if (partList != null) { @@ -244,7 +249,7 @@ public static Statistics collectStatistics(HiveConf conf, PrunedPartitionList pa List emptyStats = Lists.newArrayList(); // add partition column stats - addParitionColumnStats(neededColumns, referencedColumns, schema, table, partList, + addParitionColumnStats(conf, neededColumns, referencedColumns, schema, table, partList, emptyStats); stats.addToColumnStats(emptyStats); @@ -258,9 +263,12 @@ public static Statistics collectStatistics(HiveConf conf, PrunedPartitionList pa List columnStats = convertColStats(colStats, table.getTableName(), colToTabAlias); - addParitionColumnStats(neededColumns, referencedColumns, schema, table, partList, + addParitionColumnStats(conf, neededColumns, referencedColumns, schema, table, partList, columnStats); + // infer if any column can be primary key based on column statistics + inferAndSetPrimaryKey(stats.getNumRows(), columnStats); + stats.addToColumnStats(columnStats); State colState = deriveStatType(columnStats, referencedColumns); if (aggrStats.getPartsFound() != partNames.size() && colState != State.NONE) { @@ -275,7 +283,59 @@ public static Statistics collectStatistics(HiveConf conf, PrunedPartitionList pa return stats; } - private static void addParitionColumnStats(List neededColumns, + + /** + * Based on the provided column statistics and number of rows, this method infers if the column + * can be primary key. It checks if the difference between the min and max value is equal to + * number of rows specified. + * @param numRows - number of rows + * @param colStats - column statistics + */ + public static void inferAndSetPrimaryKey(long numRows, List colStats) { + if (colStats != null) { + for (ColStatistics cs : colStats) { + if (cs != null && cs.getRange() != null && cs.getRange().minValue != null && + cs.getRange().maxValue != null) { + if (numRows == + ((cs.getRange().maxValue.longValue() - cs.getRange().minValue.longValue()) + 1)) { + cs.setPrimaryKey(true); + } + } + } + } + } + + /** + * Infer foreign key relationship from given column statistics. + * @param csPK - column statistics of primary key + * @param csFK - column statistics of potential foreign key + * @return + */ + public static boolean inferForeignKey(ColStatistics csPK, ColStatistics csFK) { + if (csPK != null && csFK != null) { + if (csPK.isPrimaryKey()) { + if (csPK.getRange() != null && csFK.getRange() != null) { + ColStatistics.Range pkRange = csPK.getRange(); + ColStatistics.Range fkRange = csFK.getRange(); + return isWithin(fkRange, pkRange); + } + } + } + return false; + } + + private static boolean isWithin(ColStatistics.Range range1, ColStatistics.Range range2) { + if (range1.minValue != null && range2.minValue != null && range1.maxValue != null && + range2.maxValue != null) { + if (range1.minValue.longValue() >= range2.minValue.longValue() && + range1.maxValue.longValue() <= range2.maxValue.longValue()) { + return true; + } + } + return false; + } + + private static void addParitionColumnStats(HiveConf conf, List neededColumns, List referencedColumns, List schema, Table table, PrunedPartitionList partList, List colStats) throws HiveException { @@ -298,6 +358,8 @@ private static void addParitionColumnStats(List neededColumns, long numPartitions = getNDVPartitionColumn(partList.getPartitions(), ci.getInternalName()); partCS.setCountDistint(numPartitions); + partCS.setAvgColLen(StatsUtils.getAvgColLenOfVariableLengthTypes(conf, + ci.getObjectInspector(), partCS.getColumnType())); colStats.add(partCS); } } @@ -531,6 +593,7 @@ public static ColStatistics getColStatistics(ColumnStatisticsObj cso, String tab // Columns statistics for complex datatypes are not supported yet return null; } + return cs; } @@ -957,7 +1020,20 @@ public static long getWritableSize(ObjectInspector oi, Object value) { colStat.setColumnName(outColName); colStat.setTableAlias(outTabAlias); } - cs.add(colStat); + if (colStat != null) { + cs.add(colStat); + } + } + + return cs; + } + + // In cases where column expression map or row schema is missing, just pass on the parent column + // stats. This could happen in cases like TS -> FIL where FIL does not map input column names to + // internal names. + if (colExprMap == null || rowSchema == null) { + if (parentStats.getColumnStats() != null) { + cs.addAll(parentStats.getColumnStats()); } } return cs; @@ -998,7 +1074,13 @@ public static ColStatistics getColStatisticsFromExpression(HiveConf conf, Statis if (encd.getIsPartitionColOrVirtualCol()) { - // vitual columns + ColStatistics colStats = parentStats.getColumnStatisticsFromColName(colName); + if (colStats != null) { + /* If statistics for the column already exist use it. */ + return colStats; + } + + // virtual columns colType = encd.getTypeInfo().getTypeName(); countDistincts = numRows; oi = encd.getWritableObjectInspector(); @@ -1350,4 +1432,11 @@ private static void getTableAliasFromExprNode(ExprNodeDesc end, } } + + public static long getAvailableMemory(Configuration conf) { + int memory = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCONTAINERSIZE) > 0 ? + HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCONTAINERSIZE) : + conf.getInt(MRJobConfig.MAP_MEMORY_MB, MRJobConfig.DEFAULT_MAP_MEMORY_MB); + return memory; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java index 18bb2c0..146ebda 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java @@ -24,15 +24,12 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.ValidTxnList; import org.apache.hadoop.hive.common.ValidTxnListImpl; -import org.apache.hadoop.hive.metastore.api.LockComponent; -import org.apache.hadoop.hive.metastore.api.LockLevel; -import org.apache.hadoop.hive.metastore.api.LockRequest; -import org.apache.hadoop.hive.metastore.api.LockResponse; -import org.apache.hadoop.hive.metastore.api.LockState; -import org.apache.hadoop.hive.metastore.api.LockType; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.ShowLocksRequest; +import org.apache.hadoop.hive.metastore.api.ShowLocksResponse; +import org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; -import org.apache.hadoop.hive.metastore.api.UnlockRequest; import org.apache.hadoop.hive.metastore.txn.CompactionInfo; import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.security.UserGroupInformation; @@ -41,7 +38,12 @@ import java.io.IOException; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; /** * A class to clean directories after compactions. This will run in a separate thread. @@ -50,35 +52,85 @@ static final private String CLASS_NAME = Cleaner.class.getName(); static final private Log LOG = LogFactory.getLog(CLASS_NAME); - private long cleanerCheckInterval = 5000; + private long cleanerCheckInterval = 0; + + // List of compactions to clean. + private Map> compactId2LockMap = new HashMap>(); + private Map compactId2CompactInfoMap = new HashMap(); @Override public void run() { - // Make sure nothing escapes this run method and kills the metastore at large, - // so wrap it in a big catch Throwable statement. + if (cleanerCheckInterval == 0) { + cleanerCheckInterval = conf.getTimeVar( + HiveConf.ConfVars.HIVE_COMPACTOR_CLEANER_RUN_INTERVAL, TimeUnit.MILLISECONDS); + } + do { + // This is solely for testing. It checks if the test has set the looped value to false, + // and if so remembers that and then sets it to true at the end. We have to check here + // first to make sure we go through a complete iteration of the loop before resetting it. + boolean setLooped = !looped.boolVal; + // Make sure nothing escapes this run method and kills the metastore at large, + // so wrap it in a big catch Throwable statement. try { long startedAt = System.currentTimeMillis(); - // Now look for new entries ready to be cleaned. + // First look for all the compactions that are waiting to be cleaned. If we have not + // seen an entry before, look for all the locks held on that table or partition and + // record them. We will then only clean the partition once all of those locks have been + // released. This way we avoid removing the files while they are in use, + // while at the same time avoiding starving the cleaner as new readers come along. + // This works because we know that any reader who comes along after the worker thread has + // done the compaction will read the more up to date version of the data (either in a + // newer delta or in a newer base). List toClean = txnHandler.findReadyToClean(); - for (CompactionInfo ci : toClean) { - LockComponent comp = null; - comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.TABLE, ci.dbname); - comp.setTablename(ci.tableName); - if (ci.partName != null) comp.setPartitionname(ci.partName); - List components = new ArrayList(1); - components.add(comp); - LockRequest rqst = new LockRequest(components, System.getProperty("user.name"), - Worker.hostname()); - LockResponse rsp = txnHandler.lockNoWait(rqst); + if (toClean.size() > 0 || compactId2LockMap.size() > 0) { + ShowLocksResponse locksResponse = txnHandler.showLocks(new ShowLocksRequest()); + + for (CompactionInfo ci : toClean) { + // Check to see if we have seen this request before. If so, ignore it. If not, + // add it to our queue. + if (!compactId2LockMap.containsKey(ci.id)) { + compactId2LockMap.put(ci.id, findRelatedLocks(ci, locksResponse)); + compactId2CompactInfoMap.put(ci.id, ci); + } + } + + // Now, for each entry in the queue, see if all of the associated locks are clear so we + // can clean + Set currentLocks = buildCurrentLockSet(locksResponse); + List expiredLocks = new ArrayList(); + List compactionsCleaned = new ArrayList(); try { - if (rsp.getState() == LockState.ACQUIRED) { - clean(ci); + for (Map.Entry> queueEntry : compactId2LockMap.entrySet()) { + boolean sawLock = false; + for (Long lockId : queueEntry.getValue()) { + if (currentLocks.contains(lockId)) { + sawLock = true; + break; + } else { + expiredLocks.add(lockId); + } + } + + if (!sawLock) { + // Remember to remove this when we're out of the loop, + // we can't do it in the loop or we'll get a concurrent modification exception. + compactionsCleaned.add(queueEntry.getKey()); + clean(compactId2CompactInfoMap.get(queueEntry.getKey())); + } else { + // Remove the locks we didn't see so we don't look for them again next time + for (Long lockId : expiredLocks) { + queueEntry.getValue().remove(lockId); + } + } } } finally { - if (rsp.getState() == LockState.ACQUIRED) { - txnHandler.unlock(new UnlockRequest(rsp.getLockid())); + if (compactionsCleaned.size() > 0) { + for (Long compactId : compactionsCleaned) { + compactId2LockMap.remove(compactId); + compactId2CompactInfoMap.remove(compactId); + } } } } @@ -91,9 +143,37 @@ public void run() { LOG.error("Caught an exception in the main loop of compactor cleaner, " + StringUtils.stringifyException(t)); } + if (setLooped) { + looped.boolVal = true; + } } while (!stop.boolVal); } + private Set findRelatedLocks(CompactionInfo ci, ShowLocksResponse locksResponse) { + Set relatedLocks = new HashSet(); + for (ShowLocksResponseElement lock : locksResponse.getLocks()) { + if (ci.dbname.equals(lock.getDbname())) { + if ((ci.tableName == null && lock.getTablename() == null) || + (ci.tableName != null && ci.tableName.equals(lock.getTablename()))) { + if ((ci.partName == null && lock.getPartname() == null) || + (ci.partName != null && ci.partName.equals(lock.getPartname()))) { + relatedLocks.add(lock.getLockid()); + } + } + } + } + + return relatedLocks; + } + + private Set buildCurrentLockSet(ShowLocksResponse locksResponse) { + Set currentLocks = new HashSet(locksResponse.getLocks().size()); + for (ShowLocksResponseElement lock : locksResponse.getLocks()) { + currentLocks.add(lock.getLockid()); + } + return currentLocks; + } + private void clean(CompactionInfo ci) throws MetaException { LOG.info("Starting cleaning for " + ci.getFullPartitionName()); try { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java index 93d1bc0..1d3cf00 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java @@ -506,13 +506,15 @@ public void map(WritableComparable key, CompactorInputSplit split, ValidTxnList txnList = new ValidTxnListImpl(jobConf.get(ValidTxnList.VALID_TXNS_KEY)); + boolean isMajor = jobConf.getBoolean(IS_MAJOR, false); AcidInputFormat.RawReader reader = - aif.getRawReader(jobConf, jobConf.getBoolean(IS_MAJOR, false), split.getBucket(), + aif.getRawReader(jobConf, isMajor, split.getBucket(), txnList, split.getBaseDir(), split.getDeltaDirs()); RecordIdentifier identifier = reader.createKey(); V value = reader.createValue(); getWriter(reporter, reader.getObjectInspector(), split.getBucket()); while (reader.next(identifier, value)) { + if (isMajor && reader.isDelete(value)) continue; writer.write(value); reporter.progress(); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java index 715f9c0..2bba731 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java @@ -53,6 +53,7 @@ protected RawStore rs; protected int threadId; protected BooleanPointer stop; + protected BooleanPointer looped; @Override public void setHiveConf(HiveConf conf) { @@ -66,8 +67,9 @@ public void setThreadId(int threadId) { } @Override - public void init(BooleanPointer stop) throws MetaException { + public void init(BooleanPointer stop, BooleanPointer looped) throws MetaException { this.stop = stop; + this.looped = looped; setPriority(MIN_PRIORITY); setDaemon(true); // this means the process will exit without waiting for this thread diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java index c1d0fe1..5545bf7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java @@ -137,8 +137,8 @@ public void run() { } @Override - public void init(BooleanPointer stop) throws MetaException { - super.init(stop); + public void init(BooleanPointer stop, BooleanPointer looped) throws MetaException { + super.init(stop, looped); checkInterval = conf.getTimeVar(HiveConf.ConfVars.HIVE_COMPACTOR_CHECK_INTERVAL, TimeUnit.MILLISECONDS) ; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java index 249fece..e2388e7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java @@ -168,8 +168,8 @@ public Object run() throws Exception { } @Override - public void init(BooleanPointer stop) throws MetaException { - super.init(stop); + public void init(BooleanPointer stop, BooleanPointer looped) throws MetaException { + super.init(stop, looped); StringBuilder name = new StringBuilder(hostname()); name.append("-"); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java index 1f70c55..833452d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java @@ -100,6 +100,26 @@ public ReturnObjectInspectorResolver(boolean allowTypeConversion) { * @return false if there is a type mismatch */ public boolean update(ObjectInspector oi) throws UDFArgumentTypeException { + return update(oi, false); + } + + /** + * Update returnObjectInspector and valueInspectorsAreTheSame based on the + * ObjectInspector seen for UnionAll. + * + * @return false if there is a type mismatch + */ + public boolean updateForUnionAll(ObjectInspector oi) throws UDFArgumentTypeException { + return update(oi, true); + } + + /** + * Update returnObjectInspector and valueInspectorsAreTheSame based on the + * ObjectInspector seen. + * + * @return false if there is a type mismatch + */ + private boolean update(ObjectInspector oi, boolean isUnionAll) throws UDFArgumentTypeException { if (oi instanceof VoidObjectInspector) { return true; } @@ -137,8 +157,14 @@ public boolean update(ObjectInspector oi) throws UDFArgumentTypeException { // Types are different, we need to check whether we can convert them to // a common base class or not. - TypeInfo commonTypeInfo = FunctionRegistry.getCommonClass(oiTypeInfo, + TypeInfo commonTypeInfo = null; + if (isUnionAll) { + commonTypeInfo = FunctionRegistry.getCommonClassForUnionAll(oiTypeInfo, + rTypeInfo); + } else { + commonTypeInfo = FunctionRegistry.getCommonClass(oiTypeInfo, rTypeInfo); + } if (commonTypeInfo == null) { return false; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/util/ZooKeeperHiveHelper.java b/ql/src/java/org/apache/hadoop/hive/ql/util/ZooKeeperHiveHelper.java index d9faa45..29d0531 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/util/ZooKeeperHiveHelper.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/util/ZooKeeperHiveHelper.java @@ -33,6 +33,7 @@ public class ZooKeeperHiveHelper { public static final Log LOG = LogFactory.getLog(ZooKeeperHiveHelper.class.getName()); public static final String ZOOKEEPER_PATH_SEPARATOR = "/"; + /** * Get the ensemble server addresses from the configuration. The format is: host1:port, * host2:port.. @@ -90,6 +91,7 @@ public static String createPathRecursively(ZooKeeper zooKeeperClient, String pat * A no-op watcher class */ public static class DummyWatcher implements Watcher { + @Override public void process(org.apache.zookeeper.WatchedEvent event) { } } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFunctionRegistry.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFunctionRegistry.java index f2e8113..655c3d0 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFunctionRegistry.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFunctionRegistry.java @@ -256,6 +256,11 @@ public void testCommonClassComparison() { comparison(TypeInfoFactory.stringTypeInfo, TypeInfoFactory.dateTypeInfo, TypeInfoFactory.stringTypeInfo); + comparison(TypeInfoFactory.intTypeInfo, TypeInfoFactory.timestampTypeInfo, + TypeInfoFactory.doubleTypeInfo); + comparison(TypeInfoFactory.timestampTypeInfo, TypeInfoFactory.intTypeInfo, + TypeInfoFactory.doubleTypeInfo); + comparison(TypeInfoFactory.stringTypeInfo, varchar10, TypeInfoFactory.stringTypeInfo); comparison(varchar10, TypeInfoFactory.stringTypeInfo, TypeInfoFactory.stringTypeInfo); comparison(varchar5, varchar10, varchar10); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java index 36b4a43..0049f53 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java @@ -18,10 +18,7 @@ package org.apache.hadoop.hive.ql.exec; -import java.io.File; import java.io.IOException; -import java.io.OutputStream; -import java.io.PrintStream; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -32,7 +29,6 @@ import junit.framework.TestCase; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.Driver; @@ -41,7 +37,6 @@ import org.apache.hadoop.hive.ql.plan.CollectDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; -import org.apache.hadoop.hive.ql.plan.FilterDesc; import org.apache.hadoop.hive.ql.plan.MapredWork; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.PartitionDesc; @@ -49,8 +44,6 @@ import org.apache.hadoop.hive.ql.plan.ScriptDesc; import org.apache.hadoop.hive.ql.plan.SelectDesc; import org.apache.hadoop.hive.ql.plan.TableDesc; -import org.apache.hadoop.hive.ql.processors.CommandProcessor; -import org.apache.hadoop.hive.ql.processors.CommandProcessorFactory; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject; @@ -60,12 +53,9 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; -import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.RecordReader; -import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.mapred.TextInputFormat; import org.junit.Test; @@ -109,55 +99,6 @@ protected void setUp() { } } - public void testBaseFilterOperator() throws Throwable { - try { - System.out.println("Testing Filter Operator"); - ExprNodeDesc col0 = TestExecDriver.getStringColumn("col0"); - ExprNodeDesc col1 = TestExecDriver.getStringColumn("col1"); - ExprNodeDesc col2 = TestExecDriver.getStringColumn("col2"); - ExprNodeDesc zero = new ExprNodeConstantDesc("0"); - ExprNodeDesc func1 = TypeCheckProcFactory.DefaultExprProcessor - .getFuncExprNodeDesc(">", col2, col1); - ExprNodeDesc func2 = TypeCheckProcFactory.DefaultExprProcessor - .getFuncExprNodeDesc("==", col0, zero); - ExprNodeDesc func3 = TypeCheckProcFactory.DefaultExprProcessor - .getFuncExprNodeDesc("and", func1, func2); - assert (func3 != null); - FilterDesc filterCtx = new FilterDesc(func3, false); - - // Configuration - Operator op = OperatorFactory.get(FilterDesc.class); - op.setConf(filterCtx); - - // runtime initialization - op.initialize(new JobConf(TestOperators.class), - new ObjectInspector[] {r[0].oi}); - - for (InspectableObject oner : r) { - op.processOp(oner.o, 0); - } - - Map, Long> results = op.getStats(); - System.out.println("filtered = " - + results.get(FilterOperator.Counter.FILTERED)); - assertEquals(Long.valueOf(4), results - .get(FilterOperator.Counter.FILTERED)); - System.out.println("passed = " - + results.get(FilterOperator.Counter.PASSED)); - assertEquals(Long.valueOf(1), results.get(FilterOperator.Counter.PASSED)); - - /* - * for(Enum e: results.keySet()) { System.out.println(e.toString() + ":" + - * results.get(e)); } - */ - System.out.println("Filter Operator ok"); - - } catch (Throwable e) { - e.printStackTrace(); - throw e; - } - } - private void testTaskIds(String [] taskIds, String expectedAttemptId, String expectedTaskId) { Configuration conf = new JobConf(TestOperators.class); for (String one: taskIds) { @@ -328,7 +269,7 @@ public void testMapOperator() throws Throwable { try { System.out.println("Testing Map Operator"); // initialize configuration - Configuration hconf = new JobConf(TestOperators.class); + JobConf hconf = new JobConf(TestOperators.class); HiveConf.setVar(hconf, HiveConf.ConfVars.HADOOPMAPFILENAME, "hdfs:///testDir/testFile"); IOContext.get(hconf).setInputPath( diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java index c6ac557..b6834be 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java @@ -47,117 +47,122 @@ public TezSessionState createSession(String sessionId) { } @Before - public void setUp() { - conf = new HiveConf(); - } + public void setUp() { + conf = new HiveConf(); + } @Test - public void testGetNonDefaultSession() { - poolManager = new TestTezSessionPoolManager(); - try { - TezSessionState sessionState = poolManager.getSession(null, conf, true); - TezSessionState sessionState1 = poolManager.getSession(sessionState, conf, true); - if (sessionState1 != sessionState) { - fail(); - } - } catch (Exception e) { - e.printStackTrace(); + public void testGetNonDefaultSession() { + poolManager = new TestTezSessionPoolManager(); + try { + TezSessionState sessionState = poolManager.getSession(null, conf, true); + TezSessionState sessionState1 = poolManager.getSession(sessionState, conf, true); + if (sessionState1 != sessionState) { + fail(); + } + conf.set("tez.queue.name", "nondefault"); + TezSessionState sessionState2 = poolManager.getSession(sessionState, conf, true); + if (sessionState2 == sessionState) { fail(); } + } catch (Exception e) { + e.printStackTrace(); + fail(); } + } @Test - public void testSessionPoolGetInOrder() { - try { - conf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS, false); - conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_TEZ_DEFAULT_QUEUES, "a,b,c"); - conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_TEZ_SESSIONS_PER_DEFAULT_QUEUE, 2); - - poolManager = new TestTezSessionPoolManager(); - poolManager.setupPool(conf); - poolManager.startPool(); - TezSessionState sessionState = poolManager.getSession(null, conf, true); - if (sessionState.getQueueName().compareTo("a") != 0) { - fail(); - } - poolManager.returnSession(sessionState); - - sessionState = poolManager.getSession(null, conf, true); - if (sessionState.getQueueName().compareTo("b") != 0) { - fail(); - } - poolManager.returnSession(sessionState); + public void testSessionPoolGetInOrder() { + try { + conf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS, false); + conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_TEZ_DEFAULT_QUEUES, "a,b,c"); + conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_TEZ_SESSIONS_PER_DEFAULT_QUEUE, 2); - sessionState = poolManager.getSession(null, conf, true); - if (sessionState.getQueueName().compareTo("c") != 0) { - fail(); - } - poolManager.returnSession(sessionState); + poolManager = new TestTezSessionPoolManager(); + poolManager.setupPool(conf); + poolManager.startPool(); + TezSessionState sessionState = poolManager.getSession(null, conf, true); + if (sessionState.getQueueName().compareTo("a") != 0) { + fail(); + } + poolManager.returnSession(sessionState); - sessionState = poolManager.getSession(null, conf, true); - if (sessionState.getQueueName().compareTo("a") != 0) { - fail(); - } + sessionState = poolManager.getSession(null, conf, true); + if (sessionState.getQueueName().compareTo("b") != 0) { + fail(); + } + poolManager.returnSession(sessionState); - poolManager.returnSession(sessionState); + sessionState = poolManager.getSession(null, conf, true); + if (sessionState.getQueueName().compareTo("c") != 0) { + fail(); + } + poolManager.returnSession(sessionState); - } catch (Exception e) { - e.printStackTrace(); + sessionState = poolManager.getSession(null, conf, true); + if (sessionState.getQueueName().compareTo("a") != 0) { fail(); } + + poolManager.returnSession(sessionState); + + } catch (Exception e) { + e.printStackTrace(); + fail(); } + } public class SessionThread implements Runnable { @Override - public void run() { - try { - HiveConf tmpConf = new HiveConf(conf); - if (random.nextDouble() > 0.5) { - tmpConf.set("tez.queue.name", "default"); - } else { - tmpConf.set("tez.queue.name", ""); - } - - TezSessionState session = poolManager.getSession(null, tmpConf, true); - Thread.sleep((random.nextInt(9) % 10) * 1000); - poolManager.returnSession(session); - } catch (Exception e) { - e.printStackTrace(); + public void run() { + try { + HiveConf tmpConf = new HiveConf(conf); + if (random.nextDouble() > 0.5) { + tmpConf.set("tez.queue.name", "default"); + } else { + tmpConf.set("tez.queue.name", ""); } + + TezSessionState session = poolManager.getSession(null, tmpConf, true); + Thread.sleep((random.nextInt(9) % 10) * 1000); + poolManager.returnSession(session); + } catch (Exception e) { + e.printStackTrace(); } + } } @Test - public void testReturn() { - conf.set("tez.queue.name", ""); - random = new Random(1000); - conf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS, false); - conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_TEZ_DEFAULT_QUEUES, "a,b,c"); - conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_TEZ_SESSIONS_PER_DEFAULT_QUEUE, 2); + public void testReturn() { + conf.set("tez.queue.name", ""); + random = new Random(1000); + conf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS, false); + conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_TEZ_DEFAULT_QUEUES, "a,b,c"); + conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_TEZ_SESSIONS_PER_DEFAULT_QUEUE, 2); + try { + poolManager = new TestTezSessionPoolManager(); + poolManager.setupPool(conf); + poolManager.startPool(); + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + List threadList = new ArrayList(); + for (int i = 0; i < 15; i++) { + Thread t = new Thread(new SessionThread()); + t.start(); + } + + for (Thread t : threadList) { try { - poolManager = new TestTezSessionPoolManager(); - poolManager.setupPool(conf); - poolManager.startPool(); - } catch (Exception e) { + t.join(); + } catch (InterruptedException e) { e.printStackTrace(); fail(); } - List threadList = new ArrayList(); - for (int i = 0; i < 15; i++) { - Thread t = new Thread(new SessionThread()); - t.start(); - } - - for (Thread t : threadList) { - try { - t.join(); - } catch (InterruptedException e) { - e.printStackTrace(); - fail(); - } - } } + } @Test public void testCloseAndOpenDefault() throws Exception { diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionState.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionState.java index 99a7a00..63687eb 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionState.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionState.java @@ -37,9 +37,10 @@ public class TestTezSessionState extends TezSessionState { private boolean open; - private String sessionId; + private final String sessionId; private HiveConf hiveConf; private String user; + private boolean doAsEnabled; public TestTezSessionState(String sessionId) { super(sessionId); @@ -47,38 +48,46 @@ public TestTezSessionState(String sessionId) { } @Override - public boolean isOpen() { - return open; - } + public boolean isOpen() { + return open; + } public void setOpen(boolean open) { this.open = open; } @Override - public void open(HiveConf conf) throws IOException, - LoginException, URISyntaxException, TezException { - this.hiveConf = conf; - UserGroupInformation ugi; - ugi = ShimLoader.getHadoopShims().getUGIForConf(conf); - user = ShimLoader.getHadoopShims().getShortUserName(ugi); - } + public void open(HiveConf conf) throws IOException, LoginException, URISyntaxException, + TezException { + this.hiveConf = conf; + UserGroupInformation ugi; + ugi = ShimLoader.getHadoopShims().getUGIForConf(conf); + user = ShimLoader.getHadoopShims().getShortUserName(ugi); + this.doAsEnabled = conf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS); + } @Override - public void close(boolean keepTmpDir) throws TezException, IOException { - open = keepTmpDir; - } + public void close(boolean keepTmpDir) throws TezException, IOException { + open = keepTmpDir; + } + @Override public HiveConf getConf() { return this.hiveConf; } @Override - public String getSessionId() { - return sessionId; - } - + public String getSessionId() { + return sessionId; + } + + @Override public String getUser() { return user; } + + @Override + public boolean getDoAsEnabled() { + return this.doAsEnabled; + } } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorFilterOperator.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorFilterOperator.java index 24a52a6..27c1a84 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorFilterOperator.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorFilterOperator.java @@ -18,7 +18,9 @@ package org.apache.hadoop.hive.ql.exec.vector; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import junit.framework.Assert; @@ -83,9 +85,9 @@ private LongColumnVector getLongVector(int len) { private VectorFilterOperator getAVectorFilterOperator() throws HiveException { ExprNodeColumnDesc col1Expr = new ExprNodeColumnDesc(Long.class, "col1", "table", false); - Map columnMap = new HashMap(); - columnMap.put("col1", 1); - VectorizationContext vc = new VectorizationContext(columnMap, 1); + List columns = new ArrayList(); + columns.add("col1"); + VectorizationContext vc = new VectorizationContext(columns); FilterDesc fdesc = new FilterDesc(); fdesc.setPredicate(col1Expr); return new VectorFilterOperator(vc, fdesc); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java index 525aa99..c147483 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java @@ -170,10 +170,10 @@ private static GroupByDesc buildKeyGroupByDesc( @Test public void testMemoryPressureFlush() throws HiveException { - Map mapColumnNames = new HashMap(); - mapColumnNames.put("Key", 0); - mapColumnNames.put("Value", 1); - VectorizationContext ctx = new VectorizationContext(mapColumnNames, 2); + List mapColumnNames = new ArrayList(); + mapColumnNames.add("Key"); + mapColumnNames.add("Value"); + VectorizationContext ctx = new VectorizationContext(mapColumnNames); GroupByDesc desc = buildKeyGroupByDesc (ctx, "max", "Value", TypeInfoFactory.longTypeInfo, @@ -1710,7 +1710,7 @@ private void testMultiKey( mapColumnNames.put("value", i); outputColumnNames.add("value"); - VectorizationContext ctx = new VectorizationContext(mapColumnNames, i+1); + VectorizationContext ctx = new VectorizationContext(outputColumnNames); ArrayList aggs = new ArrayList(1); aggs.add( @@ -1818,10 +1818,10 @@ private void testKeyTypeAggregate( FakeVectorRowBatchFromObjectIterables data, Map expected) throws HiveException { - Map mapColumnNames = new HashMap(); - mapColumnNames.put("Key", 0); - mapColumnNames.put("Value", 1); - VectorizationContext ctx = new VectorizationContext(mapColumnNames, 2); + List mapColumnNames = new ArrayList(); + mapColumnNames.add("Key"); + mapColumnNames.add("Value"); + VectorizationContext ctx = new VectorizationContext(mapColumnNames); Set keys = new HashSet(); AggregationDesc agg = buildAggregationDesc(ctx, aggregateName, @@ -2233,9 +2233,9 @@ public static Validator getValidator(String aggregate) throws HiveException { public void testAggregateCountStarIterable ( Iterable data, Object expected) throws HiveException { - Map mapColumnNames = new HashMap(); - mapColumnNames.put("A", 0); - VectorizationContext ctx = new VectorizationContext(mapColumnNames, 1); + List mapColumnNames = new ArrayList(); + mapColumnNames.add("A"); + VectorizationContext ctx = new VectorizationContext(mapColumnNames); GroupByDesc desc = buildGroupByDescCountStar (ctx); @@ -2262,9 +2262,9 @@ public void testAggregateCountStarIterable ( public void testAggregateCountReduceIterable ( Iterable data, Object expected) throws HiveException { - Map mapColumnNames = new HashMap(); - mapColumnNames.put("A", 0); - VectorizationContext ctx = new VectorizationContext(mapColumnNames, 1); + List mapColumnNames = new ArrayList(); + mapColumnNames.add("A"); + VectorizationContext ctx = new VectorizationContext(mapColumnNames); GroupByDesc desc = buildGroupByDescType(ctx, "count", "A", TypeInfoFactory.longTypeInfo); VectorGroupByDesc vectorDesc = desc.getVectorDesc(); @@ -2294,9 +2294,9 @@ public void testAggregateStringIterable ( String aggregateName, Iterable data, Object expected) throws HiveException { - Map mapColumnNames = new HashMap(); - mapColumnNames.put("A", 0); - VectorizationContext ctx = new VectorizationContext(mapColumnNames, 1); + List mapColumnNames = new ArrayList(); + mapColumnNames.add("A"); + VectorizationContext ctx = new VectorizationContext(mapColumnNames); GroupByDesc desc = buildGroupByDescType(ctx, aggregateName, "A", TypeInfoFactory.stringTypeInfo); @@ -2325,9 +2325,9 @@ public void testAggregateDecimalIterable ( String aggregateName, Iterable data, Object expected) throws HiveException { - Map mapColumnNames = new HashMap(); - mapColumnNames.put("A", 0); - VectorizationContext ctx = new VectorizationContext(mapColumnNames, 1); + List mapColumnNames = new ArrayList(); + mapColumnNames.add("A"); + VectorizationContext ctx = new VectorizationContext(mapColumnNames); GroupByDesc desc = buildGroupByDescType(ctx, aggregateName, "A", TypeInfoFactory.getDecimalTypeInfo(30, 4)); @@ -2357,9 +2357,9 @@ public void testAggregateDoubleIterable ( String aggregateName, Iterable data, Object expected) throws HiveException { - Map mapColumnNames = new HashMap(); - mapColumnNames.put("A", 0); - VectorizationContext ctx = new VectorizationContext(mapColumnNames, 1); + List mapColumnNames = new ArrayList(); + mapColumnNames.add("A"); + VectorizationContext ctx = new VectorizationContext(mapColumnNames); GroupByDesc desc = buildGroupByDescType (ctx, aggregateName, "A", TypeInfoFactory.doubleTypeInfo); @@ -2388,9 +2388,9 @@ public void testAggregateLongIterable ( String aggregateName, Iterable data, Object expected) throws HiveException { - Map mapColumnNames = new HashMap(); - mapColumnNames.put("A", 0); - VectorizationContext ctx = new VectorizationContext(mapColumnNames, 1); + List mapColumnNames = new ArrayList(); + mapColumnNames.add("A"); + VectorizationContext ctx = new VectorizationContext(mapColumnNames); GroupByDesc desc = buildGroupByDescType(ctx, aggregateName, "A", TypeInfoFactory.longTypeInfo); @@ -2418,10 +2418,11 @@ public void testAggregateLongKeyIterable ( String aggregateName, Iterable data, HashMap expected) throws HiveException { - Map mapColumnNames = new HashMap(); - mapColumnNames.put("Key", 0); - mapColumnNames.put("Value", 1); - VectorizationContext ctx = new VectorizationContext(mapColumnNames, 2); + List mapColumnNames = new ArrayList(); + mapColumnNames.add("Key"); + mapColumnNames.add("Value"); + VectorizationContext ctx = new VectorizationContext(mapColumnNames); + Set keys = new HashSet(); GroupByDesc desc = buildKeyGroupByDesc (ctx, aggregateName, "Value", @@ -2484,10 +2485,10 @@ public void testAggregateStringKeyIterable ( Iterable data, TypeInfo dataTypeInfo, HashMap expected) throws HiveException { - Map mapColumnNames = new HashMap(); - mapColumnNames.put("Key", 0); - mapColumnNames.put("Value", 1); - VectorizationContext ctx = new VectorizationContext(mapColumnNames, 2); + List mapColumnNames = new ArrayList(); + mapColumnNames.add("Key"); + mapColumnNames.add("Value"); + VectorizationContext ctx = new VectorizationContext(mapColumnNames); Set keys = new HashSet(); GroupByDesc desc = buildKeyGroupByDesc (ctx, aggregateName, "Value", diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSelectOperator.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSelectOperator.java index 486089b..3c004a1 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSelectOperator.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSelectOperator.java @@ -84,9 +84,11 @@ public void forward(Object row, ObjectInspector rowInspector) throws HiveExcepti @Test public void testSelectOperator() throws HiveException { - Map columnMap = new HashMap(); - columnMap.put("a", 0); columnMap.put("b", 1); columnMap.put("c", 2); - VectorizationContext vc = new VectorizationContext(columnMap, 3); + List columns = new ArrayList(); + columns.add("a"); + columns.add("b"); + columns.add("c"); + VectorizationContext vc = new VectorizationContext(columns); SelectDesc selDesc = new SelectDesc(false); List colList = new ArrayList(); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java index 9e3628f..efe2efe 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java @@ -247,16 +247,16 @@ public void testArithmeticExpressionVectorization() throws HiveException { children5.add(col6Expr); modExpr.setChildren(children5); - Map columnMap = new HashMap(); - columnMap.put("col1", 1); - columnMap.put("col2", 2); - columnMap.put("col3", 3); - columnMap.put("col4", 4); - columnMap.put("col5", 5); - columnMap.put("col6", 6); + VectorizationContext vc = new VectorizationContext(); + vc.addInitialColumn("col1"); + vc.addInitialColumn("col2"); + vc.addInitialColumn("col3"); + vc.addInitialColumn("col4"); + vc.addInitialColumn("col5"); + vc.addInitialColumn("col6"); + vc.finishedAddingInitialColumns(); //Generate vectorized expression - VectorizationContext vc = new VectorizationContext(columnMap, 6); VectorExpression ve = vc.getVectorExpression(sumExpr, VectorExpressionDescriptor.Mode.PROJECTION); //Verify vectorized expression @@ -293,11 +293,11 @@ public void testStringFilterExpressions() throws HiveException { children1.add(constDesc); exprDesc.setChildren(children1); - Map columnMap = new HashMap(); - columnMap.put("col1", 1); - columnMap.put("col2", 2); - - VectorizationContext vc = new VectorizationContext(columnMap, 2); + List columns = new ArrayList(); + columns.add("col0"); + columns.add("col1"); + columns.add("col2"); + VectorizationContext vc = new VectorizationContext(columns); VectorExpression ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); @@ -318,11 +318,11 @@ public void testFilterStringColCompareStringColumnExpressions() throws HiveExcep children1.add(col2Expr); exprDesc.setChildren(children1); - Map columnMap = new HashMap(); - columnMap.put("col1", 1); - columnMap.put("col2", 2); - - VectorizationContext vc = new VectorizationContext(columnMap, 2); + List columns = new ArrayList(); + columns.add("col0"); + columns.add("col1"); + columns.add("col2"); + VectorizationContext vc = new VectorizationContext(columns); VectorExpression ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); @@ -341,7 +341,7 @@ public void testFilterStringColCompareStringColumnExpressions() throws HiveExcep children1.add(col2Expr); exprDesc.setChildren(children1); - vc = new VectorizationContext(columnMap, 2); + vc = new VectorizationContext(columns); ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); @@ -360,7 +360,7 @@ public void testFilterStringColCompareStringColumnExpressions() throws HiveExcep children1.add(col2Expr); exprDesc.setChildren(children1); - vc = new VectorizationContext(columnMap, 2); + vc = new VectorizationContext(columns); ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); @@ -378,7 +378,7 @@ public void testFilterStringColCompareStringColumnExpressions() throws HiveExcep children1.add(col2Expr); exprDesc.setChildren(children1); - vc = new VectorizationContext(columnMap, 2); + vc = new VectorizationContext(columns); ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); @@ -395,7 +395,7 @@ public void testFilterStringColCompareStringColumnExpressions() throws HiveExcep children1.add(col2Expr); exprDesc.setChildren(children1); - vc = new VectorizationContext(columnMap, 2); + vc = new VectorizationContext(columns); ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); @@ -412,7 +412,7 @@ public void testFilterStringColCompareStringColumnExpressions() throws HiveExcep children1.add(col2Expr); exprDesc.setChildren(children1); - vc = new VectorizationContext(columnMap, 2); + vc = new VectorizationContext(columns); ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); @@ -432,10 +432,9 @@ public void testFloatInExpressions() throws HiveException { ExprNodeGenericFuncDesc exprDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.doubleTypeInfo, udf, children1); - Map columnMap = new HashMap(); - columnMap.put("col1", 0); - - VectorizationContext vc = new VectorizationContext(columnMap, 2); + List columns = new ArrayList(); + columns.add("col1"); + VectorizationContext vc = new VectorizationContext(columns); VectorExpression ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.PROJECTION); @@ -477,11 +476,11 @@ public void testVectorizeFilterAndOrExpression() throws HiveException { children3.add(lessExprDesc); andExprDesc.setChildren(children3); - Map columnMap = new HashMap(); - columnMap.put("col1", 0); - columnMap.put("col2", 1); - - VectorizationContext vc = new VectorizationContext(columnMap, 2); + List columns = new ArrayList(); + columns.add("col0"); + columns.add("col1"); + columns.add("col2"); + VectorizationContext vc = new VectorizationContext(columns); VectorExpression ve = vc.getVectorExpression(andExprDesc, VectorExpressionDescriptor.Mode.FILTER); @@ -528,11 +527,10 @@ public void testVectorizeAndOrProjectionExpression() throws HiveException { children3.add(col2Expr); andExprDesc.setChildren(children3); - Map columnMap = new HashMap(); - columnMap.put("col1", 0); - columnMap.put("col2", 1); - - VectorizationContext vc = new VectorizationContext(columnMap, 2); + List columns = new ArrayList(); + columns.add("col1"); + columns.add("col2"); + VectorizationContext vc = new VectorizationContext(columns); VectorExpression veAnd = vc.getVectorExpression(andExprDesc, VectorExpressionDescriptor.Mode.FILTER); assertEquals(veAnd.getClass(), FilterExprAndExpr.class); assertEquals(veAnd.getChildExpressions()[0].getClass(), FilterLongColGreaterLongScalar.class); @@ -557,7 +555,7 @@ public void testVectorizeAndOrProjectionExpression() throws HiveException { orExprDesc.setChildren(children4); //Allocate new Vectorization context to reset the intermediate columns. - vc = new VectorizationContext(columnMap, 2); + vc = new VectorizationContext(columns); VectorExpression veOr = vc.getVectorExpression(orExprDesc, VectorExpressionDescriptor.Mode.FILTER); assertEquals(veOr.getClass(), FilterExprOrExpr.class); assertEquals(veOr.getChildExpressions()[0].getClass(), FilterLongColGreaterLongScalar.class); @@ -594,11 +592,11 @@ public void testNotExpression() throws HiveException { childOfNot.add(greaterExprDesc); notExpr.setChildren(childOfNot); - Map columnMap = new HashMap(); - columnMap.put("col1", 0); - columnMap.put("col2", 1); - - VectorizationContext vc = new VectorizationContext(columnMap, 2); + List columns = new ArrayList(); + columns.add("col0"); + columns.add("col1"); + columns.add("col2"); + VectorizationContext vc = new VectorizationContext(columns); VectorExpression ve = vc.getVectorExpression(notExpr, VectorExpressionDescriptor.Mode.FILTER); @@ -632,11 +630,10 @@ public void testNullExpressions() throws HiveException { childOfIsNull.add(greaterExprDesc); isNullExpr.setChildren(childOfIsNull); - Map columnMap = new HashMap(); - columnMap.put("col1", 0); - columnMap.put("col2", 1); - - VectorizationContext vc = new VectorizationContext(columnMap, 2); + List columns = new ArrayList(); + columns.add("col1"); + columns.add("col2"); + VectorizationContext vc = new VectorizationContext(columns); VectorExpression ve = vc.getVectorExpression(isNullExpr, VectorExpressionDescriptor.Mode.FILTER); @@ -674,11 +671,10 @@ public void testNotNullExpressions() throws HiveException { childOfNot.add(greaterExprDesc); isNotNullExpr.setChildren(childOfNot); - Map columnMap = new HashMap(); - columnMap.put("col1", 0); - columnMap.put("col2", 1); - - VectorizationContext vc = new VectorizationContext(columnMap, 2); + List columns = new ArrayList(); + columns.add("col1"); + columns.add("col2"); + VectorizationContext vc = new VectorizationContext(columns); VectorExpression ve = vc.getVectorExpression(isNotNullExpr, VectorExpressionDescriptor.Mode.FILTER); @@ -705,10 +701,9 @@ public void testVectorizeScalarColumnExpression() throws HiveException { ExprNodeGenericFuncDesc scalarMinusConstant = new ExprNodeGenericFuncDesc(TypeInfoFactory.longTypeInfo, gudf, children); - Map columnMap = new HashMap(); - columnMap.put("a", 0); - - VectorizationContext vc = new VectorizationContext(columnMap, 2); + List columns = new ArrayList(); + columns.add("a"); + VectorizationContext vc = new VectorizationContext(columns); VectorExpression ve = vc.getVectorExpression(scalarMinusConstant, VectorExpressionDescriptor.Mode.PROJECTION); assertEquals(ve.getClass(), LongScalarSubtractLongColumn.class); @@ -727,11 +722,11 @@ public void testFilterWithNegativeScalar() throws HiveException { children1.add(constDesc); exprDesc.setChildren(children1); - Map columnMap = new HashMap(); - columnMap.put("col1", 1); - columnMap.put("col2", 2); - - VectorizationContext vc = new VectorizationContext(columnMap, 2); + List columns = new ArrayList(); + columns.add("col0"); + columns.add("col1"); + columns.add("col2"); + VectorizationContext vc = new VectorizationContext(columns); VectorExpression ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); @@ -746,9 +741,10 @@ public void testUnaryMinusColumnLong() throws HiveException { children.add(col1Expr); ExprNodeGenericFuncDesc negExprDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.longTypeInfo, gudf, children); - Map columnMap = new HashMap(); - columnMap.put("col1", 1); - VectorizationContext vc = new VectorizationContext(columnMap, 1); + List columns = new ArrayList(); + columns.add("col0"); + columns.add("col1"); + VectorizationContext vc = new VectorizationContext(columns); VectorExpression ve = vc.getVectorExpression(negExprDesc, VectorExpressionDescriptor.Mode.PROJECTION); @@ -763,9 +759,10 @@ public void testUnaryMinusColumnDouble() throws HiveException { children.add(col1Expr); ExprNodeGenericFuncDesc negExprDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.doubleTypeInfo, gudf, children); - Map columnMap = new HashMap(); - columnMap.put("col1", 1); - VectorizationContext vc = new VectorizationContext(columnMap, 1); + List columns = new ArrayList(); + columns.add("col0"); + columns.add("col1"); + VectorizationContext vc = new VectorizationContext(columns); VectorExpression ve = vc.getVectorExpression(negExprDesc, VectorExpressionDescriptor.Mode.PROJECTION); @@ -788,10 +785,9 @@ public void testFilterScalarCompareColumn() throws HiveException { scalarGreaterColExpr.setChildren(children); - Map columnMap = new HashMap(); - columnMap.put("a", 0); - - VectorizationContext vc = new VectorizationContext(columnMap, 2); + List columns = new ArrayList(); + columns.add("a"); + VectorizationContext vc = new VectorizationContext(columns); VectorExpression ve = vc.getVectorExpression(scalarGreaterColExpr, VectorExpressionDescriptor.Mode.FILTER); assertEquals(FilterLongScalarGreaterLongColumn.class, ve.getClass()); } @@ -812,10 +808,9 @@ public void testFilterBooleanColumnCompareBooleanScalar() throws HiveException { colEqualScalar.setChildren(children); - Map columnMap = new HashMap(); - columnMap.put("a", 0); - - VectorizationContext vc = new VectorizationContext(columnMap, 2); + List columns = new ArrayList(); + columns.add("a"); + VectorizationContext vc = new VectorizationContext(columns); VectorExpression ve = vc.getVectorExpression(colEqualScalar, VectorExpressionDescriptor.Mode.FILTER); assertEquals(FilterLongColEqualLongScalar.class, ve.getClass()); } @@ -836,9 +831,9 @@ public void testBooleanColumnCompareBooleanScalar() throws HiveException { colEqualScalar.setChildren(children); - Map columnMap = new HashMap(); - columnMap.put("a", 0); - VectorizationContext vc = new VectorizationContext(columnMap, 2); + List columns = new ArrayList(); + columns.add("a"); + VectorizationContext vc = new VectorizationContext(columns); VectorExpression ve = vc.getVectorExpression(colEqualScalar, VectorExpressionDescriptor.Mode.PROJECTION); assertEquals(LongColEqualLongScalar.class, ve.getClass()); } @@ -852,11 +847,10 @@ public void testUnaryStringExpressions() throws HiveException { children.add(colDesc); stringUnary.setChildren(children); - Map columnMap = new HashMap(); - columnMap.put("b", 0); - columnMap.put("a", 1); - VectorizationContext vc = new VectorizationContext(columnMap, 2); - + List columns = new ArrayList(); + columns.add("b"); + columns.add("a"); + VectorizationContext vc = new VectorizationContext(columns); GenericUDF stringLower = new GenericUDFLower(); stringUnary.setGenericUDF(stringLower); @@ -866,7 +860,7 @@ public void testUnaryStringExpressions() throws HiveException { assertEquals(1, ((StringLower) ve).getColNum()); assertEquals(2, ((StringLower) ve).getOutputColumn()); - vc = new VectorizationContext(columnMap, 2); + vc = new VectorizationContext(columns); ExprNodeGenericFuncDesc anotherUnary = new ExprNodeGenericFuncDesc(); anotherUnary.setTypeInfo(TypeInfoFactory.stringTypeInfo); @@ -898,10 +892,10 @@ public void testMathFunctions() throws HiveException { children1.add(colDesc1); children2.add(colDesc2); - Map columnMap = new HashMap(); - columnMap.put("b", 0); - columnMap.put("a", 1); - VectorizationContext vc = new VectorizationContext(columnMap, 2); + List columns = new ArrayList(); + columns.add("b"); + columns.add("a"); + VectorizationContext vc = new VectorizationContext(columns); // Sin(double) GenericUDFBridge gudfBridge = new GenericUDFBridge("sin", false, UDFSin.class.getName()); @@ -989,10 +983,10 @@ public void testTimeStampUdfs() throws HiveException { List children = new ArrayList(); children.add(colDesc1); - Map columnMap = new HashMap(); - columnMap.put("b", 0); - columnMap.put("a", 1); - VectorizationContext vc = new VectorizationContext(columnMap, 2); + List columns = new ArrayList(); + columns.add("b"); + columns.add("a"); + VectorizationContext vc = new VectorizationContext(columns); //UDFYear GenericUDFBridge gudfBridge = new GenericUDFBridge("year", false, UDFYear.class.getName()); @@ -1026,10 +1020,12 @@ public void testBetweenFilters() throws HiveException { ExprNodeGenericFuncDesc exprDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, udf, children1); - Map columnMap = new HashMap(); - columnMap.put("col1", 1); - columnMap.put("col2", 2); - VectorizationContext vc = new VectorizationContext(columnMap, 2); + List columns = new ArrayList(); + columns.add("col0"); + columns.add("col1"); + columns.add("col2"); + VectorizationContext vc = new VectorizationContext(columns); + VectorExpression ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); assertTrue(ve instanceof FilterStringColumnBetween); @@ -1054,7 +1050,7 @@ public void testBetweenFilters() throws HiveException { exprDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, udf, children1); - vc = new VectorizationContext(columnMap, 2); + vc = new VectorizationContext(columns); ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); assertTrue(ve instanceof FilterCharColumnBetween); @@ -1079,7 +1075,7 @@ public void testBetweenFilters() throws HiveException { exprDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, udf, children1); - vc = new VectorizationContext(columnMap, 2); + vc = new VectorizationContext(columns); ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); assertTrue(ve instanceof FilterVarCharColumnBetween); @@ -1144,10 +1140,11 @@ public void testInFiltersAndExprs() throws HiveException { ExprNodeGenericFuncDesc exprDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, udf, children1); - Map columnMap = new HashMap(); - columnMap.put("col1", 1); - columnMap.put("col2", 2); - VectorizationContext vc = new VectorizationContext(columnMap, 2); + List columns = new ArrayList(); + columns.add("col0"); + columns.add("col1"); + columns.add("col2"); + VectorizationContext vc = new VectorizationContext(columns); VectorExpression ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); assertTrue(ve instanceof FilterStringColumnInList); ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.PROJECTION); @@ -1197,11 +1194,12 @@ public void testIfConditionalExprs() throws HiveException { ExprNodeGenericFuncDesc exprDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, udf, children1); - Map columnMap = new HashMap(); - columnMap.put("col1", 1); - columnMap.put("col2", 2); - columnMap.put("col3", 3); - VectorizationContext vc = new VectorizationContext(columnMap, 3); + List columns = new ArrayList(); + columns.add("col0"); + columns.add("col1"); + columns.add("col2"); + columns.add("col3"); + VectorizationContext vc = new VectorizationContext(columns); VectorExpression ve = vc.getVectorExpression(exprDesc); assertTrue(ve instanceof IfExprLongColumnLongColumn); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java b/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java index b174496..c0bdb00 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java @@ -217,11 +217,11 @@ public void testObsoleteOriginals() throws Exception { Path part = new MockPath(fs, "/tbl/part1"); AcidUtils.Directory dir = AcidUtils.getAcidState(part, conf, new ValidTxnListImpl("150:")); + // The two original buckets won't be in the obsolete list because we don't look at those + // until we have determined there is no base. List obsolete = dir.getObsolete(); - assertEquals(3, obsolete.size()); + assertEquals(1, obsolete.size()); assertEquals("mock:/tbl/part1/base_5", obsolete.get(0).getPath().toString()); - assertEquals("mock:/tbl/part1/000000_0", obsolete.get(1).getPath().toString()); - assertEquals("mock:/tbl/part1/000001_1", obsolete.get(2).getPath().toString()); assertEquals("mock:/tbl/part1/base_10", dir.getBaseDirectory().toString()); } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java index 55392c9..3635fc7 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java @@ -73,6 +73,7 @@ import org.apache.hadoop.hive.ql.plan.MapWork; import org.apache.hadoop.hive.ql.plan.PartitionDesc; import org.apache.hadoop.hive.ql.plan.TableDesc; +import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; import org.apache.hadoop.hive.serde2.SerDe; import org.apache.hadoop.hive.serde2.SerDeUtils; @@ -1266,6 +1267,8 @@ JobConf createMockExecutionEnvironment(Path workDir, } conf.set("hive.io.file.readcolumn.ids", columnIds.toString()); conf.set("partition_columns", "p"); + conf.set(serdeConstants.LIST_COLUMNS, columnNames.toString()); + conf.set(serdeConstants.LIST_COLUMN_TYPES, columnTypes.toString()); MockFileSystem fs = (MockFileSystem) warehouseDir.getFileSystem(conf); fs.clear(); @@ -1295,8 +1298,8 @@ JobConf createMockExecutionEnvironment(Path workDir, } mapWork.setPathToAliases(aliasMap); mapWork.setPathToPartitionInfo(partMap); - mapWork.setScratchColumnMap(new HashMap>()); - mapWork.setScratchColumnVectorTypes(new HashMap>()); + mapWork.setAllScratchColumnVectorTypeMaps(new HashMap>()); // write the plan out diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java index 421fbda..595e003 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java @@ -56,6 +56,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; public class TestOrcRawRecordMerger { @@ -574,12 +575,14 @@ public void testNewBaseAndDelta() throws Exception { OrcRecordUpdater.getOperation(event)); assertEquals(new ReaderKey(0, BUCKET, 0, 200), id); assertEquals("update 1", getValue(event)); + assertFalse(merger.isDelete(event)); assertEquals(true, merger.next(id, event)); assertEquals(OrcRecordUpdater.INSERT_OPERATION, OrcRecordUpdater.getOperation(event)); assertEquals(new ReaderKey(0, BUCKET, 1, 0), id); assertEquals("second", getValue(event)); + assertFalse(merger.isDelete(event)); assertEquals(true, merger.next(id, event)); assertEquals(OrcRecordUpdater.UPDATE_OPERATION, @@ -616,6 +619,7 @@ public void testNewBaseAndDelta() throws Exception { OrcRecordUpdater.getOperation(event)); assertEquals(new ReaderKey(0, BUCKET, 7, 200), id); assertNull(OrcRecordUpdater.getRow(event)); + assertTrue(merger.isDelete(event)); assertEquals(true, merger.next(id, event)); assertEquals(OrcRecordUpdater.DELETE_OPERATION, diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapredParquetOutputFormat.java b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapredParquetOutputFormat.java index 417676d..667d367 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapredParquetOutputFormat.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapredParquetOutputFormat.java @@ -73,7 +73,8 @@ protected ParquetRecordWriterWrapper getParquerRecordWriterWrapper( ParquetOutputFormat realOutputFormat, JobConf jobConf, String finalOutPath, - Progressable progress + Progressable progress, + Properties tableProperties ) throws IOException { assertEquals(outputFormat, realOutputFormat); assertNotNull(jobConf.get(DataWritableWriteSupport.PARQUET_HIVE_SCHEMA)); @@ -87,4 +88,17 @@ protected ParquetRecordWriterWrapper getParquerRecordWriterWrapper( assertEquals("passed tests", e.getMessage()); } } + + @Test(expected = IllegalArgumentException.class) + public void testInvalidCompressionTableProperties() throws IOException { + Properties tableProps = new Properties(); + tableProps.setProperty("parquet.compression", "unsupported"); + tableProps.setProperty("columns", "foo,bar"); + tableProps.setProperty("columns.types", "int:int"); + + JobConf jobConf = new JobConf(); + + new MapredParquetOutputFormat().getHiveRecordWriter(jobConf, + new Path("/foo"), null, false, tableProps, null); + } } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java b/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java index 96728d9..831ef8c 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java @@ -2881,4 +2881,24 @@ public void testBuilderComplexTypes2() throws Exception { "leaf-3 = (NULL_SAFE_EQUALS a stinger)\n" + "expr = (and (not leaf-0) (not leaf-1) (not leaf-2) (not leaf-3))", sarg.toString()); } + + @Test + public void testBuilderFloat() throws Exception { + SearchArgument sarg = + SearchArgumentFactory.newBuilder() + .startAnd() + .lessThan("x", new Short((short) 22)) + .lessThan("x1", new Integer(22)) + .lessThanEquals("y", new HiveChar("hi", 10)) + .equals("z", new Float("0.22")) + .equals("z1", new Double(0.22)) + .end() + .build(); + assertEquals("leaf-0 = (LESS_THAN x 22)\n" + + "leaf-1 = (LESS_THAN x1 22)\n" + + "leaf-2 = (LESS_THAN_EQUALS y hi)\n" + + "leaf-3 = (EQUALS z 0.22)\n" + + "leaf-4 = (EQUALS z1 0.22)\n" + + "expr = (and leaf-0 leaf-1 leaf-2 leaf-3 leaf-4)", sarg.toString()); + } } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java index 153908c..da7d082 100755 --- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java @@ -550,7 +550,7 @@ public void testIndex() throws Throwable { index.getIndexName()); assertEquals("Table names don't match for index: " + indexName, tableName, index.getOrigTableName()); - assertEquals("Index table names didn't match for index: " + indexName, indexTableName, + assertEquals("Index table names didn't match for index: " + indexName, qIndexTableName, index.getIndexTableName()); assertEquals("Index handler classes didn't match for index: " + indexName, indexHandlerClass, index.getIndexHandlerClass()); @@ -564,7 +564,7 @@ public void testIndex() throws Throwable { // Drop index try { - hm.dropIndex(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName, indexName, true); + hm.dropIndex(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName, indexName, false, true); } catch (HiveException e) { System.err.println(StringUtils.stringifyException(e)); assertTrue("Unable to drop index: " + indexName, false); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java index fbab3af..3271189 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java @@ -45,13 +45,14 @@ @Before public void setUp() { - Map columnMap = new HashMap(); - columnMap.put("col1", 0); - columnMap.put("col2", 1); - columnMap.put("col3", 2); + List columns = new ArrayList(); + columns.add("col0"); + columns.add("col1"); + columns.add("col2"); + columns.add("col3"); //Generate vectorized expression - vContext = new VectorizationContext(columnMap, 3); + vContext = new VectorizationContext(columns); } @Description(name = "fake", value = "FAKE") diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestColumnAccess.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestColumnAccess.java new file mode 100644 index 0000000..5d22e27 --- /dev/null +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestColumnAccess.java @@ -0,0 +1,193 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.parse; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +import junit.framework.Assert; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.CommandNeedRetryException; +import org.apache.hadoop.hive.ql.Driver; +import org.apache.hadoop.hive.ql.QueryPlan; +import org.apache.hadoop.hive.ql.hooks.ReadEntity; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestColumnAccess { + + @BeforeClass + public static void Setup() throws CommandNeedRetryException { + Driver driver = createDriver(); + int ret = driver.run("create table t1(id1 int, name1 string)").getResponseCode(); + Assert.assertEquals("Checking command success", 0, ret); + ret = driver.run("create table t2(id2 int, id1 int, name2 string)").getResponseCode(); + Assert.assertEquals("Checking command success", 0, ret); + ret = driver.run("create view v1 as select * from t1").getResponseCode(); + Assert.assertEquals("Checking command success", 0, ret); + } + + @AfterClass + public static void Teardown() throws Exception { + Driver driver = createDriver(); + driver.run("drop table t1"); + driver.run("drop table t2"); + driver.run("drop view v1"); + } + + @Test + public void testQueryTable1() throws ParseException { + String query = "select * from t1"; + Driver driver = createDriver(); + int rc = driver.compile(query); + Assert.assertEquals("Checking command success", 0, rc); + QueryPlan plan = driver.getPlan(); + // check access columns from ColumnAccessInfo + ColumnAccessInfo columnAccessInfo = plan.getColumnAccessInfo(); + List cols = columnAccessInfo.getTableToColumnAccessMap().get("default@t1"); + Assert.assertNotNull(cols); + Assert.assertEquals(2, cols.size()); + Assert.assertNotNull(cols.contains("id1")); + Assert.assertNotNull(cols.contains("name1")); + + // check access columns from readEntity + Map> tableColsMap = getColsFromReadEntity(plan.getInputs()); + cols = tableColsMap.get("default@t1"); + Assert.assertNotNull(cols); + Assert.assertEquals(2, cols.size()); + Assert.assertNotNull(cols.contains("id1")); + Assert.assertNotNull(cols.contains("name1")); + } + + @Test + public void testJoinTable1AndTable2() throws ParseException { + String query = "select * from t1 join t2 on (t1.id1 = t2.id1)"; + Driver driver = createDriver(); + int rc = driver.compile(query); + Assert.assertEquals("Checking command success", 0, rc); + QueryPlan plan = driver.getPlan(); + // check access columns from ColumnAccessInfo + ColumnAccessInfo columnAccessInfo = plan.getColumnAccessInfo(); + List cols = columnAccessInfo.getTableToColumnAccessMap().get("default@t1"); + Assert.assertNotNull(cols); + Assert.assertEquals(2, cols.size()); + Assert.assertNotNull(cols.contains("id1")); + Assert.assertNotNull(cols.contains("name1")); + cols = columnAccessInfo.getTableToColumnAccessMap().get("default@t2"); + Assert.assertNotNull(cols); + Assert.assertEquals(3, cols.size()); + Assert.assertNotNull(cols.contains("id2")); + Assert.assertNotNull(cols.contains("id1")); + Assert.assertNotNull(cols.contains("name1")); + + + // check access columns from readEntity + Map> tableColsMap = getColsFromReadEntity(plan.getInputs()); + cols = tableColsMap.get("default@t1"); + Assert.assertNotNull(cols); + Assert.assertEquals(2, cols.size()); + Assert.assertNotNull(cols.contains("id1")); + Assert.assertNotNull(cols.contains("name1")); + cols = tableColsMap.get("default@t2"); + Assert.assertNotNull(cols); + Assert.assertEquals(3, cols.size()); + Assert.assertNotNull(cols.contains("id2")); + Assert.assertNotNull(cols.contains("id1")); + Assert.assertNotNull(cols.contains("name1")); + } + + @Test + public void testJoinView1AndTable2() throws ParseException { + String query = "select * from v1 join t2 on (v1.id1 = t2.id1)"; + Driver driver = createDriver(); + int rc = driver.compile(query); + Assert.assertEquals("Checking command success", 0, rc); + QueryPlan plan = driver.getPlan(); + // check access columns from ColumnAccessInfo + ColumnAccessInfo columnAccessInfo = plan.getColumnAccessInfo(); + List cols = columnAccessInfo.getTableToColumnAccessMap().get("default@v1"); + Assert.assertNull(cols); + cols = columnAccessInfo.getTableToColumnAccessMap().get("default@t1"); + Assert.assertNotNull(cols); + Assert.assertEquals(2, cols.size()); + Assert.assertNotNull(cols.contains("id1")); + Assert.assertNotNull(cols.contains("name1")); + cols = columnAccessInfo.getTableToColumnAccessMap().get("default@t2"); + Assert.assertNotNull(cols); + Assert.assertEquals(3, cols.size()); + Assert.assertNotNull(cols.contains("id2")); + Assert.assertNotNull(cols.contains("id1")); + Assert.assertNotNull(cols.contains("name1")); + + + // check access columns from readEntity + Map> tableColsMap = getColsFromReadEntity(plan.getInputs()); + cols = tableColsMap.get("default@v1"); + Assert.assertNull(cols); + cols = tableColsMap.get("default@t1"); + Assert.assertNotNull(cols); + Assert.assertEquals(2, cols.size()); + Assert.assertNotNull(cols.contains("id1")); + Assert.assertNotNull(cols.contains("name1")); + cols = tableColsMap.get("default@t2"); + Assert.assertNotNull(cols); + Assert.assertEquals(3, cols.size()); + Assert.assertNotNull(cols.contains("id2")); + Assert.assertNotNull(cols.contains("id1")); + Assert.assertNotNull(cols.contains("name1")); + } + + private Map> getColsFromReadEntity(HashSet inputs) { + Map> tableColsMap = new HashMap>(); + for(ReadEntity entity: inputs) { + switch (entity.getType()) { + case TABLE: + if (entity.getAccessedColumns() != null && !entity.getAccessedColumns().isEmpty()) { + tableColsMap.put(entity.getTable().getCompleteName(), entity.getAccessedColumns()); + } + break; + case PARTITION: + if (entity.getAccessedColumns() != null && !entity.getAccessedColumns().isEmpty()) { + tableColsMap.put(entity.getPartition().getTable().getCompleteName(), + entity.getAccessedColumns()); + } + break; + default: + // no-op + } + } + return tableColsMap; + } + + private static Driver createDriver() { + HiveConf conf = new HiveConf(Driver.class); + HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); + conf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_COLLECT_SCANCOLS, true); + SessionState.start(conf); + Driver driver = new Driver(conf); + driver.init(); + return driver; + } + +} diff --git a/ql/src/test/org/apache/hadoop/hive/ql/plan/TestReadEntityDirect.java b/ql/src/test/org/apache/hadoop/hive/ql/plan/TestReadEntityDirect.java index 2d27cec..6615a45 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/plan/TestReadEntityDirect.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/plan/TestReadEntityDirect.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hive.ql.session.SessionState; import org.junit.Before; import org.junit.BeforeClass; +import org.junit.AfterClass; import org.junit.Test; /** @@ -56,6 +57,13 @@ public static void onetimeSetup() throws CommandNeedRetryException { assertEquals("Checking command success", 0, ret); } + @AfterClass + public static void onetimeTeardown() throws Exception { + Driver driver = createDriver(); + driver.run("drop table t1"); + driver.run("drop view v1"); + } + @Before public void setup() { CheckInputReadEntityDirect.readEntities = null; diff --git a/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/TestSQLStdHiveAccessControllerCLI.java b/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/TestSQLStdHiveAccessControllerCLI.java index 5c4d713..ac862c5 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/TestSQLStdHiveAccessControllerCLI.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/TestSQLStdHiveAccessControllerCLI.java @@ -55,13 +55,10 @@ public void testConfigProcessing() throws HiveAuthzPluginException { assertFalse("Check for transform query disabling hook", processedConf.getVar(ConfVars.PREEXECHOOKS).contains(DisallowTransformHook.class.getName())); - // check that set param whitelist is not set - assertTrue(processedConf.getVar(ConfVars.HIVE_AUTHORIZATION_SQL_STD_AUTH_CONFIG_WHITELIST) == null - || processedConf.getVar(ConfVars.HIVE_AUTHORIZATION_SQL_STD_AUTH_CONFIG_WHITELIST).trim() - .equals("")); - // verify that some dummy param can be set processedConf.verifyAndSet("dummy.param", "dummy.val"); + processedConf.verifyAndSet(ConfVars.HIVE_AUTHORIZATION_ENABLED.varname, "true"); + } private HiveAuthzSessionContext getCLISessionCtx() { diff --git a/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/TestSQLStdHiveAccessControllerHS2.java b/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/TestSQLStdHiveAccessControllerHS2.java index f13cf7e..d6872a0 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/TestSQLStdHiveAccessControllerHS2.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/TestSQLStdHiveAccessControllerHS2.java @@ -20,6 +20,11 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.ql.security.HadoopDefaultAuthenticator; @@ -42,20 +47,53 @@ * policy on hiveconf correctly * * @throws HiveAuthzPluginException + * @throws IllegalAccessException + * @throws NoSuchFieldException + * @throws IllegalArgumentException + * @throws SecurityException */ @Test - public void testConfigProcessing() throws HiveAuthzPluginException { - HiveConf processedConf = new HiveConf(); + public void testConfigProcessing() throws HiveAuthzPluginException, SecurityException, + IllegalArgumentException, NoSuchFieldException, IllegalAccessException { + HiveConf processedConf = newAuthEnabledConf(); SQLStdHiveAccessController accessController = new SQLStdHiveAccessController(null, - processedConf, new HadoopDefaultAuthenticator(), getHS2SessionCtx() - ); + processedConf, new HadoopDefaultAuthenticator(), getHS2SessionCtx()); accessController.applyAuthorizationConfigPolicy(processedConf); // check that hook to disable transforms has been added assertTrue("Check for transform query disabling hook", processedConf.getVar(ConfVars.PREEXECHOOKS).contains(DisallowTransformHook.class.getName())); - verifyParamSettability(SQLStdHiveAccessController.defaultModWhiteListSqlStdAuth, processedConf); + List settableParams = getSettableParams(); + verifyParamSettability(settableParams, processedConf); + + } + + private HiveConf newAuthEnabledConf() { + HiveConf conf = new HiveConf(); + conf.setBoolVar(ConfVars.HIVE_AUTHORIZATION_ENABLED, true); + return conf; + } + + /** + * @return list of parameters that should be possible to set + */ + private List getSettableParams() throws SecurityException, NoSuchFieldException, + IllegalArgumentException, IllegalAccessException { + // get all the variable names being converted to regex in HiveConf, using reflection + Field varNameField = HiveConf.class.getDeclaredField("sqlStdAuthSafeVarNames"); + varNameField.setAccessible(true); + List confVarList = Arrays.asList((String[]) varNameField.get(null)); + + // create list with variables that match some of the regexes + List confVarRegexList = Arrays.asList("hive.convert.join.bucket.mapjoin.tez", + "hive.optimize.index.filter.compact.maxsize", "hive.tez.dummy", "tez.task.dummy"); + + // combine two lists + List varList = new ArrayList(); + varList.addAll(confVarList); + varList.addAll(confVarRegexList); + return varList; } @@ -70,7 +108,7 @@ private HiveAuthzSessionContext getHS2SessionCtx() { * @param settableParams * @param processedConf */ - private void verifyParamSettability(String [] settableParams, HiveConf processedConf) { + private void verifyParamSettability(List settableParams, HiveConf processedConf) { // verify that the whitlelist params can be set for (String param : settableParams) { try { @@ -90,24 +128,42 @@ private void verifyParamSettability(String [] settableParams, HiveConf processed } /** - * Test that modifying HIVE_AUTHORIZATION_SQL_STD_AUTH_CONFIG_WHITELIST config works + * Test that setting HIVE_AUTHORIZATION_SQL_STD_AUTH_CONFIG_WHITELIST_APPEND config works + * @throws HiveAuthzPluginException + */ + @Test + public void testConfigProcessingCustomSetWhitelistAppend() throws HiveAuthzPluginException { + // append new config params to whitelist + List paramRegexes = Arrays.asList("hive.ctest.param", "hive.abc..*"); + List settableParams = Arrays.asList("hive.ctest.param", "hive.abc.def"); + verifySettability(paramRegexes, settableParams, + ConfVars.HIVE_AUTHORIZATION_SQL_STD_AUTH_CONFIG_WHITELIST_APPEND); + } + + /** + * Test that setting HIVE_AUTHORIZATION_SQL_STD_AUTH_CONFIG_WHITELIST config works * @throws HiveAuthzPluginException */ @Test public void testConfigProcessingCustomSetWhitelist() throws HiveAuthzPluginException { + // append new config params to whitelist + List paramRegexes = Arrays.asList("hive.ctest.param", "hive.abc..*"); + List settableParams = Arrays.asList("hive.ctest.param", "hive.abc.def"); + verifySettability(paramRegexes, settableParams, + ConfVars.HIVE_AUTHORIZATION_SQL_STD_AUTH_CONFIG_WHITELIST); + } - HiveConf processedConf = new HiveConf(); - // add custom value, including one from the default, one new one - String[] settableParams = { SQLStdHiveAccessController.defaultModWhiteListSqlStdAuth[0], - "abcs.dummy.test.param" }; - processedConf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_SQL_STD_AUTH_CONFIG_WHITELIST, - Joiner.on(",").join(settableParams)); + private void verifySettability(List paramRegexes, List settableParams, + ConfVars whiteListParam) throws HiveAuthzPluginException { + HiveConf processedConf = newAuthEnabledConf(); + processedConf.setVar(whiteListParam, + Joiner.on("|").join(paramRegexes)); SQLStdHiveAccessController accessController = new SQLStdHiveAccessController(null, processedConf, new HadoopDefaultAuthenticator(), getHS2SessionCtx()); accessController.applyAuthorizationConfigPolicy(processedConf); - verifyParamSettability(settableParams, processedConf); + verifyParamSettability(settableParams, processedConf); } private void assertConfModificationException(HiveConf processedConf, String param) { @@ -120,4 +176,5 @@ private void assertConfModificationException(HiveConf processedConf, String para assertTrue("Exception should be thrown while modifying the param " + param, caughtEx); } + } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java index ec1379d..9ebdfd3 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java @@ -63,12 +63,13 @@ protected CompactionTxnHandler txnHandler; protected IMetaStoreClient ms; protected long sleepTime = 1000; + protected HiveConf conf; private final MetaStoreThread.BooleanPointer stop = new MetaStoreThread.BooleanPointer(); private final File tmpdir; protected CompactorTest() throws Exception { - HiveConf conf = new HiveConf(); + conf = new HiveConf(); TxnDbUtil.setConfValues(conf); TxnDbUtil.cleanDb(); ms = new HiveMetaStoreClient(conf); @@ -79,16 +80,20 @@ protected CompactorTest() throws Exception { tmpdir.deleteOnExit(); } - protected void startInitiator(HiveConf conf) throws Exception { - startThread('i', conf); + protected void startInitiator() throws Exception { + startThread('i', true); } - protected void startWorker(HiveConf conf) throws Exception { - startThread('w', conf); + protected void startWorker() throws Exception { + startThread('w', true); } - protected void startCleaner(HiveConf conf) throws Exception { - startThread('c', conf); + protected void startCleaner() throws Exception { + startThread('c', true); + } + + protected void startCleaner(MetaStoreThread.BooleanPointer looped) throws Exception { + startThread('c', false, looped); } protected Table newTable(String dbName, String tableName, boolean partitioned) throws TException { @@ -117,6 +122,9 @@ protected Table newTable(String dbName, String tableName, boolean partitioned, table.setParameters(parameters); + // drop the table first, in case some previous test created it + ms.dropTable(dbName, tableName); + ms.createTable(table); return table; } @@ -142,37 +150,27 @@ protected long openTxn() throws MetaException { return txns.get(0); } - protected void addDeltaFile(HiveConf conf, Table t, Partition p, long minTxn, long maxTxn, - int numRecords) throws Exception{ - addFile(conf, t, p, minTxn, maxTxn, numRecords, FileType.DELTA, 2, true); - } - - protected void addBaseFile(HiveConf conf, Table t, Partition p, long maxTxn, - int numRecords) throws Exception{ - addFile(conf, t, p, 0, maxTxn, numRecords, FileType.BASE, 2, true); + protected void addDeltaFile(Table t, Partition p, long minTxn, long maxTxn, int numRecords) + throws Exception { + addFile(t, p, minTxn, maxTxn, numRecords, FileType.DELTA, 2, true); } - protected void addLegacyFile(HiveConf conf, Table t, Partition p, - int numRecords) throws Exception { - addFile(conf, t, p, 0, 0, numRecords, FileType.LEGACY, 2, true); + protected void addBaseFile(Table t, Partition p, long maxTxn, int numRecords) throws Exception { + addFile(t, p, 0, maxTxn, numRecords, FileType.BASE, 2, true); } - protected void addDeltaFile(HiveConf conf, Table t, Partition p, long minTxn, long maxTxn, - int numRecords, int numBuckets, boolean allBucketsPresent) - throws Exception { - addFile(conf, t, p, minTxn, maxTxn, numRecords, FileType.DELTA, numBuckets, allBucketsPresent); + protected void addLegacyFile(Table t, Partition p, int numRecords) throws Exception { + addFile(t, p, 0, 0, numRecords, FileType.LEGACY, 2, true); } - protected void addBaseFile(HiveConf conf, Table t, Partition p, long maxTxn, - int numRecords, int numBuckets, boolean allBucketsPresent) - throws Exception { - addFile(conf, t, p, 0, maxTxn, numRecords, FileType.BASE, numBuckets, allBucketsPresent); + protected void addDeltaFile(Table t, Partition p, long minTxn, long maxTxn, int numRecords, + int numBuckets, boolean allBucketsPresent) throws Exception { + addFile(t, p, minTxn, maxTxn, numRecords, FileType.DELTA, numBuckets, allBucketsPresent); } - protected void addLegacyFile(HiveConf conf, Table t, Partition p, - int numRecords, int numBuckets, boolean allBucketsPresent) - throws Exception { - addFile(conf, t, p, 0, 0, numRecords, FileType.LEGACY, numBuckets, allBucketsPresent); + protected void addBaseFile(Table t, Partition p, long maxTxn, int numRecords, int numBuckets, + boolean allBucketsPresent) throws Exception { + addFile(t, p, 0, maxTxn, numRecords, FileType.BASE, numBuckets, allBucketsPresent); } protected List getDirectories(HiveConf conf, Table t, Partition p) throws Exception { @@ -191,6 +189,10 @@ protected void burnThroughTransactions(int num) throws MetaException, NoSuchTxnE for (long tid : rsp.getTxn_ids()) txnHandler.commitTxn(new CommitTxnRequest(tid)); } + protected void stopThread() { + stop.boolVal = true; + } + private StorageDescriptor newStorageDescriptor(String location, List sortCols) { StorageDescriptor sd = new StorageDescriptor(); List cols = new ArrayList(2); @@ -214,9 +216,13 @@ private StorageDescriptor newStorageDescriptor(String location, List sort return sd; } - // I can't do this with @Before because I want to be able to control the config file provided - // to each test. - private void startThread(char type, HiveConf conf) throws Exception { + // I can't do this with @Before because I want to be able to control when the thead starts + private void startThread(char type, boolean stopAfterOne) throws Exception { + startThread(type, stopAfterOne, new MetaStoreThread.BooleanPointer()); + } + + private void startThread(char type, boolean stopAfterOne, MetaStoreThread.BooleanPointer looped) + throws Exception { TxnDbUtil.setConfValues(conf); CompactorThread t = null; switch (type) { @@ -227,9 +233,10 @@ private void startThread(char type, HiveConf conf) throws Exception { } t.setThreadId((int) t.getId()); t.setHiveConf(conf); - stop.boolVal = true; - t.init(stop); - t.run(); + stop.boolVal = stopAfterOne; + t.init(stop, looped); + if (stopAfterOne) t.run(); + else t.start(); } private String getLocation(String tableName, String partValue) { @@ -243,7 +250,7 @@ private String getLocation(String tableName, String partValue) { private enum FileType {BASE, DELTA, LEGACY}; - private void addFile(HiveConf conf, Table t, Partition p, long minTxn, long maxTxn, + private void addFile(Table t, Partition p, long minTxn, long maxTxn, int numRecords, FileType type, int numBuckets, boolean allBucketsPresent) throws Exception { String partValue = (p == null) ? null : p.getValues().get(0); @@ -332,6 +339,7 @@ public boolean validateInput(FileSystem fs, HiveConf conf, ArrayList private final Configuration conf; private FSDataInputStream is = null; private final FileSystem fs; + private boolean lastWasDelete = true; MockRawReader(Configuration conf, List files) throws IOException { filesToRead = new Stack(); @@ -346,6 +354,15 @@ public ObjectInspector getObjectInspector() { } @Override + public boolean isDelete(Text value) { + // Alternate between returning deleted and not. This is easier than actually + // tracking operations. We test that this is getting properly called by checking that only + // half the records show up in base files after major compactions. + lastWasDelete = !lastWasDelete; + return lastWasDelete; + } + + @Override public boolean next(RecordIdentifier identifier, Text text) throws IOException { if (is == null) { // Open the next file diff --git a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestCleaner.java b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestCleaner.java index edc7f32..b63ad66 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestCleaner.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestCleaner.java @@ -18,21 +18,26 @@ package org.apache.hadoop.hive.ql.txn.compactor; import junit.framework.Assert; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.MetaStoreThread; import org.apache.hadoop.hive.metastore.api.*; import org.apache.hadoop.hive.metastore.txn.CompactionInfo; -import org.apache.hadoop.hive.metastore.txn.TxnDbUtil; -import org.junit.Before; import org.junit.Test; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.TimeUnit; /** * Tests for the compactor Cleaner thread */ public class TestCleaner extends CompactorTest { + + static final private Log LOG = LogFactory.getLog(TestCleaner.class.getName()); + public TestCleaner() throws Exception { super(); } @@ -41,19 +46,17 @@ public TestCleaner() throws Exception { public void nothing() throws Exception { // Test that the whole things works when there's nothing in the queue. This is just a // survival test. - startCleaner(new HiveConf()); + startCleaner(); } @Test public void cleanupAfterMajorTableCompaction() throws Exception { Table t = newTable("default", "camtc", false); - HiveConf conf = new HiveConf(); - - addBaseFile(conf, t, null, 20L, 20); - addDeltaFile(conf, t, null, 21L, 22L, 2); - addDeltaFile(conf, t, null, 23L, 24L, 2); - addBaseFile(conf, t, null, 25L, 25); + addBaseFile(t, null, 20L, 20); + addDeltaFile(t, null, 21L, 22L, 2); + addDeltaFile(t, null, 23L, 24L, 2); + addBaseFile(t, null, 25L, 25); burnThroughTransactions(25); @@ -63,7 +66,7 @@ public void cleanupAfterMajorTableCompaction() throws Exception { txnHandler.markCompacted(ci); txnHandler.setRunAs(ci.id, System.getProperty("user.name")); - startCleaner(conf); + startCleaner(); // Check there are no compactions requests left. ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); @@ -80,12 +83,10 @@ public void cleanupAfterMajorPartitionCompaction() throws Exception { Table t = newTable("default", "campc", true); Partition p = newPartition(t, "today"); - HiveConf conf = new HiveConf(); - - addBaseFile(conf, t, p, 20L, 20); - addDeltaFile(conf, t, p, 21L, 22L, 2); - addDeltaFile(conf, t, p, 23L, 24L, 2); - addBaseFile(conf, t, p, 25L, 25); + addBaseFile(t, p, 20L, 20); + addDeltaFile(t, p, 21L, 22L, 2); + addDeltaFile(t, p, 23L, 24L, 2); + addBaseFile(t, p, 25L, 25); burnThroughTransactions(25); @@ -96,7 +97,7 @@ public void cleanupAfterMajorPartitionCompaction() throws Exception { txnHandler.markCompacted(ci); txnHandler.setRunAs(ci.id, System.getProperty("user.name")); - startCleaner(conf); + startCleaner(); // Check there are no compactions requests left. ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); @@ -112,12 +113,10 @@ public void cleanupAfterMajorPartitionCompaction() throws Exception { public void cleanupAfterMinorTableCompaction() throws Exception { Table t = newTable("default", "camitc", false); - HiveConf conf = new HiveConf(); - - addBaseFile(conf, t, null, 20L, 20); - addDeltaFile(conf, t, null, 21L, 22L, 2); - addDeltaFile(conf, t, null, 23L, 24L, 2); - addDeltaFile(conf, t, null, 21L, 24L, 4); + addBaseFile(t, null, 20L, 20); + addDeltaFile(t, null, 21L, 22L, 2); + addDeltaFile(t, null, 23L, 24L, 2); + addDeltaFile(t, null, 21L, 24L, 4); burnThroughTransactions(25); @@ -127,7 +126,7 @@ public void cleanupAfterMinorTableCompaction() throws Exception { txnHandler.markCompacted(ci); txnHandler.setRunAs(ci.id, System.getProperty("user.name")); - startCleaner(conf); + startCleaner(); // Check there are no compactions requests left. ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); @@ -151,12 +150,10 @@ public void cleanupAfterMinorPartitionCompaction() throws Exception { Table t = newTable("default", "camipc", true); Partition p = newPartition(t, "today"); - HiveConf conf = new HiveConf(); - - addBaseFile(conf, t, p, 20L, 20); - addDeltaFile(conf, t, p, 21L, 22L, 2); - addDeltaFile(conf, t, p, 23L, 24L, 2); - addDeltaFile(conf, t, p, 21L, 24L, 4); + addBaseFile(t, p, 20L, 20); + addDeltaFile(t, p, 21L, 22L, 2); + addDeltaFile(t, p, 23L, 24L, 2); + addDeltaFile(t, p, 21L, 24L, 4); burnThroughTransactions(25); @@ -167,7 +164,7 @@ public void cleanupAfterMinorPartitionCompaction() throws Exception { txnHandler.markCompacted(ci); txnHandler.setRunAs(ci.id, System.getProperty("user.name")); - startCleaner(conf); + startCleaner(); // Check there are no compactions requests left. ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); @@ -190,12 +187,10 @@ public void cleanupAfterMinorPartitionCompaction() throws Exception { public void blockedByLockTable() throws Exception { Table t = newTable("default", "bblt", false); - HiveConf conf = new HiveConf(); - - addBaseFile(conf, t, null, 20L, 20); - addDeltaFile(conf, t, null, 21L, 22L, 2); - addDeltaFile(conf, t, null, 23L, 24L, 2); - addDeltaFile(conf, t, null, 21L, 24L, 4); + addBaseFile(t, null, 20L, 20); + addDeltaFile(t, null, 21L, 22L, 2); + addDeltaFile(t, null, 23L, 24L, 2); + addDeltaFile(t, null, 21L, 24L, 4); burnThroughTransactions(25); @@ -212,7 +207,7 @@ public void blockedByLockTable() throws Exception { LockRequest req = new LockRequest(components, "me", "localhost"); LockResponse res = txnHandler.lock(req); - startCleaner(conf); + startCleaner(); // Check there are no compactions requests left. ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); @@ -228,12 +223,10 @@ public void blockedByLockPartition() throws Exception { Table t = newTable("default", "bblp", true); Partition p = newPartition(t, "today"); - HiveConf conf = new HiveConf(); - - addBaseFile(conf, t, p, 20L, 20); - addDeltaFile(conf, t, p, 21L, 22L, 2); - addDeltaFile(conf, t, p, 23L, 24L, 2); - addDeltaFile(conf, t, p, 21L, 24L, 4); + addBaseFile(t, p, 20L, 20); + addDeltaFile(t, p, 21L, 22L, 2); + addDeltaFile(t, p, 23L, 24L, 2); + addDeltaFile(t, p, 21L, 24L, 4); burnThroughTransactions(25); @@ -244,7 +237,7 @@ public void blockedByLockPartition() throws Exception { txnHandler.markCompacted(ci); txnHandler.setRunAs(ci.id, System.getProperty("user.name")); - LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, "default"); + LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.PARTITION, "default"); comp.setTablename("bblp"); comp.setPartitionname("ds=today"); List components = new ArrayList(1); @@ -252,7 +245,7 @@ public void blockedByLockPartition() throws Exception { LockRequest req = new LockRequest(components, "me", "localhost"); LockResponse res = txnHandler.lock(req); - startCleaner(conf); + startCleaner(); // Check there are no compactions requests left. ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); @@ -265,15 +258,154 @@ public void blockedByLockPartition() throws Exception { } @Test + public void notBlockedBySubsequentLock() throws Exception { + Table t = newTable("default", "bblt", false); + + // Set the run frequency low on this test so it doesn't take long + conf.setTimeVar(HiveConf.ConfVars.HIVE_COMPACTOR_CLEANER_RUN_INTERVAL, 100, + TimeUnit.MILLISECONDS); + + addBaseFile(t, null, 20L, 20); + addDeltaFile(t, null, 21L, 22L, 2); + addDeltaFile(t, null, 23L, 24L, 2); + addDeltaFile(t, null, 21L, 24L, 4); + + burnThroughTransactions(25); + + CompactionRequest rqst = new CompactionRequest("default", "bblt", CompactionType.MINOR); + txnHandler.compact(rqst); + CompactionInfo ci = txnHandler.findNextToCompact("fred"); + txnHandler.markCompacted(ci); + txnHandler.setRunAs(ci.id, System.getProperty("user.name")); + + LockComponent comp = new LockComponent(LockType.SHARED_READ, LockLevel.TABLE, "default"); + comp.setTablename("bblt"); + List components = new ArrayList(1); + components.add(comp); + LockRequest req = new LockRequest(components, "me", "localhost"); + LockResponse res = txnHandler.lock(req); + + MetaStoreThread.BooleanPointer looped = new MetaStoreThread.BooleanPointer(); + looped.boolVal = false; + startCleaner(looped); + + // Make sure the compactor has a chance to run once + while (!looped.boolVal) { + Thread.currentThread().sleep(100); + } + + // There should still be one request, as the locks still held. + ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); + List compacts = rsp.getCompacts(); + Assert.assertEquals(1, compacts.size()); + + // obtain a second lock. This shouldn't block cleaner as it was acquired after the initial + // clean request + LockComponent comp2 = new LockComponent(LockType.SHARED_READ, LockLevel.TABLE, "default"); + comp2.setTablename("bblt"); + List components2 = new ArrayList(1); + components2.add(comp2); + LockRequest req2 = new LockRequest(components, "me", "localhost"); + LockResponse res2 = txnHandler.lock(req2); + + // Unlock the previous lock + txnHandler.unlock(new UnlockRequest(res.getLockid())); + looped.boolVal = false; + + while (!looped.boolVal) { + Thread.currentThread().sleep(100); + } + stopThread(); + Thread.currentThread().sleep(200); + + + // Check there are no compactions requests left. + rsp = txnHandler.showCompact(new ShowCompactRequest()); + compacts = rsp.getCompacts(); + Assert.assertEquals(0, compacts.size()); + } + + @Test + public void partitionNotBlockedBySubsequentLock() throws Exception { + Table t = newTable("default", "bblt", true); + Partition p = newPartition(t, "today"); + + // Set the run frequency low on this test so it doesn't take long + conf.setTimeVar(HiveConf.ConfVars.HIVE_COMPACTOR_CLEANER_RUN_INTERVAL, 100, + TimeUnit.MILLISECONDS); + + addBaseFile(t, p, 20L, 20); + addDeltaFile(t, p, 21L, 22L, 2); + addDeltaFile(t, p, 23L, 24L, 2); + addDeltaFile(t, p, 21L, 24L, 4); + + burnThroughTransactions(25); + + CompactionRequest rqst = new CompactionRequest("default", "bblt", CompactionType.MINOR); + rqst.setPartitionname("ds=today"); + txnHandler.compact(rqst); + CompactionInfo ci = txnHandler.findNextToCompact("fred"); + txnHandler.markCompacted(ci); + txnHandler.setRunAs(ci.id, System.getProperty("user.name")); + + LockComponent comp = new LockComponent(LockType.SHARED_READ, LockLevel.PARTITION, "default"); + comp.setTablename("bblt"); + comp.setPartitionname("ds=today"); + List components = new ArrayList(1); + components.add(comp); + LockRequest req = new LockRequest(components, "me", "localhost"); + LockResponse res = txnHandler.lock(req); + + MetaStoreThread.BooleanPointer looped = new MetaStoreThread.BooleanPointer(); + looped.boolVal = false; + startCleaner(looped); + + // Make sure the compactor has a chance to run once + while (!looped.boolVal) { + Thread.currentThread().sleep(100); + } + + // There should still be one request, as the locks still held. + ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); + List compacts = rsp.getCompacts(); + Assert.assertEquals(1, compacts.size()); + + + // obtain a second lock. This shouldn't block cleaner as it was acquired after the initial + // clean request + LockComponent comp2 = new LockComponent(LockType.SHARED_READ, LockLevel.PARTITION, "default"); + comp2.setTablename("bblt"); + comp2.setPartitionname("ds=today"); + List components2 = new ArrayList(1); + components2.add(comp2); + LockRequest req2 = new LockRequest(components, "me", "localhost"); + LockResponse res2 = txnHandler.lock(req2); + + // Unlock the previous lock + txnHandler.unlock(new UnlockRequest(res.getLockid())); + looped.boolVal = false; + + while (!looped.boolVal) { + Thread.currentThread().sleep(100); + } + stopThread(); + Thread.currentThread().sleep(200); + + + // Check there are no compactions requests left. + rsp = txnHandler.showCompact(new ShowCompactRequest()); + compacts = rsp.getCompacts(); + Assert.assertEquals(0, compacts.size()); + } + + @Test public void cleanupAfterMajorPartitionCompactionNoBase() throws Exception { Table t = newTable("default", "campcnb", true); Partition p = newPartition(t, "today"); - HiveConf conf = new HiveConf(); - - addDeltaFile(conf, t, p, 1L, 22L, 22); - addDeltaFile(conf, t, p, 23L, 24L, 2); - addBaseFile(conf, t, p, 25L, 25); + addDeltaFile(t, p, 1L, 22L, 22); + addDeltaFile(t, p, 23L, 24L, 2); + addBaseFile(t, p, 25L, 25); burnThroughTransactions(25); @@ -284,7 +416,7 @@ public void cleanupAfterMajorPartitionCompactionNoBase() throws Exception { txnHandler.markCompacted(ci); txnHandler.setRunAs(ci.id, System.getProperty("user.name")); - startCleaner(conf); + startCleaner(); // Check there are no compactions requests left. ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); @@ -295,9 +427,4 @@ public void cleanupAfterMajorPartitionCompactionNoBase() throws Exception { Assert.assertEquals(1, paths.size()); Assert.assertEquals("base_25", paths.get(0).getName()); } - - @Before - public void setUpTxnDb() throws Exception { - TxnDbUtil.setConfValues(new HiveConf()); - } } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java index 128fef1..7d88c1a 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java @@ -47,7 +47,7 @@ public TestInitiator() throws Exception { public void nothing() throws Exception { // Test that the whole things works when there's nothing in the queue. This is just a // survival test. - startInitiator(new HiveConf()); + startInitiator(); } @Test @@ -63,7 +63,7 @@ public void recoverFailedLocalWorkers() throws Exception { txnHandler.findNextToCompact(Worker.hostname() + "-193892"); txnHandler.findNextToCompact("nosuchhost-193892"); - startInitiator(new HiveConf()); + startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List compacts = rsp.getCompacts(); @@ -89,10 +89,9 @@ public void recoverFailedRemoteWorkers() throws Exception { txnHandler.findNextToCompact("nosuchhost-193892"); - HiveConf conf = new HiveConf(); conf.setTimeVar(HiveConf.ConfVars.HIVE_COMPACTOR_WORKER_TIMEOUT, 1L, TimeUnit.MILLISECONDS); - startInitiator(conf); + startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List compacts = rsp.getCompacts(); @@ -104,7 +103,6 @@ public void recoverFailedRemoteWorkers() throws Exception { public void majorCompactOnTableTooManyAborts() throws Exception { Table t = newTable("default", "mcottma", false); - HiveConf conf = new HiveConf(); HiveConf.setIntVar(conf, HiveConf.ConfVars.HIVE_COMPACTOR_ABORTEDTXN_THRESHOLD, 10); for (int i = 0; i < 11; i++) { @@ -119,7 +117,7 @@ public void majorCompactOnTableTooManyAborts() throws Exception { txnHandler.abortTxn(new AbortTxnRequest(txnid)); } - startInitiator(conf); + startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List compacts = rsp.getCompacts(); @@ -134,7 +132,6 @@ public void majorCompactOnPartitionTooManyAborts() throws Exception { Table t = newTable("default", "mcoptma", true); Partition p = newPartition(t, "today"); - HiveConf conf = new HiveConf(); HiveConf.setIntVar(conf, HiveConf.ConfVars.HIVE_COMPACTOR_ABORTEDTXN_THRESHOLD, 10); for (int i = 0; i < 11; i++) { @@ -150,7 +147,7 @@ public void majorCompactOnPartitionTooManyAborts() throws Exception { txnHandler.abortTxn(new AbortTxnRequest(txnid)); } - startInitiator(conf); + startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List compacts = rsp.getCompacts(); @@ -168,7 +165,6 @@ public void noCompactOnManyDifferentPartitionAborts() throws Exception { Partition p = newPartition(t, "day-" + i); } - HiveConf conf = new HiveConf(); HiveConf.setIntVar(conf, HiveConf.ConfVars.HIVE_COMPACTOR_ABORTEDTXN_THRESHOLD, 10); for (int i = 0; i < 11; i++) { @@ -184,7 +180,7 @@ public void noCompactOnManyDifferentPartitionAborts() throws Exception { txnHandler.abortTxn(new AbortTxnRequest(txnid)); } - startInitiator(conf); + startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); Assert.assertEquals(0, rsp.getCompactsSize()); @@ -197,8 +193,6 @@ public void cleanEmptyAbortedTxns() throws Exception { // accidently clean it too. Table t = newTable("default", "ceat", false); - HiveConf conf = new HiveConf(); - long txnid = openTxn(); LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, "default"); comp.setTablename("ceat"); @@ -216,7 +210,7 @@ public void cleanEmptyAbortedTxns() throws Exception { GetOpenTxnsResponse openTxns = txnHandler.getOpenTxns(); Assert.assertEquals(101, openTxns.getOpen_txnsSize()); - startInitiator(conf); + startInitiator(); openTxns = txnHandler.getOpenTxns(); Assert.assertEquals(1, openTxns.getOpen_txnsSize()); @@ -228,7 +222,6 @@ public void noCompactWhenNoCompactSet() throws Exception { parameters.put("NO_AUTO_COMPACTION", "true"); Table t = newTable("default", "ncwncs", false, parameters); - HiveConf conf = new HiveConf(); HiveConf.setIntVar(conf, HiveConf.ConfVars.HIVE_COMPACTOR_ABORTEDTXN_THRESHOLD, 10); for (int i = 0; i < 11; i++) { @@ -243,7 +236,7 @@ public void noCompactWhenNoCompactSet() throws Exception { txnHandler.abortTxn(new AbortTxnRequest(txnid)); } - startInitiator(conf); + startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); Assert.assertEquals(0, rsp.getCompactsSize()); @@ -253,7 +246,6 @@ public void noCompactWhenNoCompactSet() throws Exception { public void noCompactWhenCompactAlreadyScheduled() throws Exception { Table t = newTable("default", "ncwcas", false); - HiveConf conf = new HiveConf(); HiveConf.setIntVar(conf, HiveConf.ConfVars.HIVE_COMPACTOR_ABORTEDTXN_THRESHOLD, 10); for (int i = 0; i < 11; i++) { @@ -277,7 +269,7 @@ public void noCompactWhenCompactAlreadyScheduled() throws Exception { Assert.assertEquals("initiated", compacts.get(0).getState()); Assert.assertEquals("ncwcas", compacts.get(0).getTablename()); - startInitiator(conf); + startInitiator(); rsp = txnHandler.showCompact(new ShowCompactRequest()); compacts = rsp.getCompacts(); @@ -291,11 +283,9 @@ public void noCompactWhenCompactAlreadyScheduled() throws Exception { public void compactTableHighDeltaPct() throws Exception { Table t = newTable("default", "cthdp", false); - HiveConf conf = new HiveConf(); - - addBaseFile(conf, t, null, 20L, 20); - addDeltaFile(conf, t, null, 21L, 22L, 2); - addDeltaFile(conf, t, null, 23L, 24L, 2); + addBaseFile(t, null, 20L, 20); + addDeltaFile(t, null, 21L, 22L, 2); + addDeltaFile(t, null, 23L, 24L, 2); burnThroughTransactions(23); @@ -309,7 +299,7 @@ public void compactTableHighDeltaPct() throws Exception { LockResponse res = txnHandler.lock(req); txnHandler.commitTxn(new CommitTxnRequest(txnid)); - startInitiator(conf); + startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List compacts = rsp.getCompacts(); @@ -324,11 +314,9 @@ public void compactPartitionHighDeltaPct() throws Exception { Table t = newTable("default", "cphdp", true); Partition p = newPartition(t, "today"); - HiveConf conf = new HiveConf(); - - addBaseFile(conf, t, p, 20L, 20); - addDeltaFile(conf, t, p, 21L, 22L, 2); - addDeltaFile(conf, t, p, 23L, 24L, 2); + addBaseFile(t, p, 20L, 20); + addDeltaFile(t, p, 21L, 22L, 2); + addDeltaFile(t, p, 23L, 24L, 2); burnThroughTransactions(23); @@ -343,7 +331,7 @@ public void compactPartitionHighDeltaPct() throws Exception { LockResponse res = txnHandler.lock(req); txnHandler.commitTxn(new CommitTxnRequest(txnid)); - startInitiator(conf); + startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List compacts = rsp.getCompacts(); @@ -358,11 +346,9 @@ public void compactPartitionHighDeltaPct() throws Exception { public void noCompactTableDeltaPctNotHighEnough() throws Exception { Table t = newTable("default", "nctdpnhe", false); - HiveConf conf = new HiveConf(); - - addBaseFile(conf, t, null, 50L, 50); - addDeltaFile(conf, t, null, 21L, 22L, 2); - addDeltaFile(conf, t, null, 23L, 24L, 2); + addBaseFile(t, null, 50L, 50); + addDeltaFile(t, null, 21L, 22L, 2); + addDeltaFile(t, null, 23L, 24L, 2); burnThroughTransactions(53); @@ -376,7 +362,7 @@ public void noCompactTableDeltaPctNotHighEnough() throws Exception { LockResponse res = txnHandler.lock(req); txnHandler.commitTxn(new CommitTxnRequest(txnid)); - startInitiator(conf); + startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); Assert.assertEquals(0, rsp.getCompactsSize()); @@ -386,20 +372,18 @@ public void noCompactTableDeltaPctNotHighEnough() throws Exception { public void compactTableTooManyDeltas() throws Exception { Table t = newTable("default", "cttmd", false); - HiveConf conf = new HiveConf(); - - addBaseFile(conf, t, null, 200L, 200); - addDeltaFile(conf, t, null, 201L, 201L, 1); - addDeltaFile(conf, t, null, 202L, 202L, 1); - addDeltaFile(conf, t, null, 203L, 203L, 1); - addDeltaFile(conf, t, null, 204L, 204L, 1); - addDeltaFile(conf, t, null, 205L, 205L, 1); - addDeltaFile(conf, t, null, 206L, 206L, 1); - addDeltaFile(conf, t, null, 207L, 207L, 1); - addDeltaFile(conf, t, null, 208L, 208L, 1); - addDeltaFile(conf, t, null, 209L, 209L, 1); - addDeltaFile(conf, t, null, 210L, 210L, 1); - addDeltaFile(conf, t, null, 211L, 211L, 1); + addBaseFile(t, null, 200L, 200); + addDeltaFile(t, null, 201L, 201L, 1); + addDeltaFile(t, null, 202L, 202L, 1); + addDeltaFile(t, null, 203L, 203L, 1); + addDeltaFile(t, null, 204L, 204L, 1); + addDeltaFile(t, null, 205L, 205L, 1); + addDeltaFile(t, null, 206L, 206L, 1); + addDeltaFile(t, null, 207L, 207L, 1); + addDeltaFile(t, null, 208L, 208L, 1); + addDeltaFile(t, null, 209L, 209L, 1); + addDeltaFile(t, null, 210L, 210L, 1); + addDeltaFile(t, null, 211L, 211L, 1); burnThroughTransactions(210); @@ -413,7 +397,7 @@ public void compactTableTooManyDeltas() throws Exception { LockResponse res = txnHandler.lock(req); txnHandler.commitTxn(new CommitTxnRequest(txnid)); - startInitiator(conf); + startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List compacts = rsp.getCompacts(); @@ -428,20 +412,18 @@ public void compactPartitionTooManyDeltas() throws Exception { Table t = newTable("default", "cptmd", true); Partition p = newPartition(t, "today"); - HiveConf conf = new HiveConf(); - - addBaseFile(conf, t, p, 200L, 200); - addDeltaFile(conf, t, p, 201L, 201L, 1); - addDeltaFile(conf, t, p, 202L, 202L, 1); - addDeltaFile(conf, t, p, 203L, 203L, 1); - addDeltaFile(conf, t, p, 204L, 204L, 1); - addDeltaFile(conf, t, p, 205L, 205L, 1); - addDeltaFile(conf, t, p, 206L, 206L, 1); - addDeltaFile(conf, t, p, 207L, 207L, 1); - addDeltaFile(conf, t, p, 208L, 208L, 1); - addDeltaFile(conf, t, p, 209L, 209L, 1); - addDeltaFile(conf, t, p, 210L, 210L, 1); - addDeltaFile(conf, t, p, 211L, 211L, 1); + addBaseFile(t, p, 200L, 200); + addDeltaFile(t, p, 201L, 201L, 1); + addDeltaFile(t, p, 202L, 202L, 1); + addDeltaFile(t, p, 203L, 203L, 1); + addDeltaFile(t, p, 204L, 204L, 1); + addDeltaFile(t, p, 205L, 205L, 1); + addDeltaFile(t, p, 206L, 206L, 1); + addDeltaFile(t, p, 207L, 207L, 1); + addDeltaFile(t, p, 208L, 208L, 1); + addDeltaFile(t, p, 209L, 209L, 1); + addDeltaFile(t, p, 210L, 210L, 1); + addDeltaFile(t, p, 211L, 211L, 1); burnThroughTransactions(210); @@ -456,7 +438,7 @@ public void compactPartitionTooManyDeltas() throws Exception { LockResponse res = txnHandler.lock(req); txnHandler.commitTxn(new CommitTxnRequest(txnid)); - startInitiator(conf); + startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List compacts = rsp.getCompacts(); @@ -471,11 +453,9 @@ public void compactPartitionTooManyDeltas() throws Exception { public void noCompactTableNotEnoughDeltas() throws Exception { Table t = newTable("default", "nctned", false); - HiveConf conf = new HiveConf(); - - addBaseFile(conf, t, null, 200L, 200); - addDeltaFile(conf, t, null, 201L, 205L, 5); - addDeltaFile(conf, t, null, 206L, 211L, 6); + addBaseFile(t, null, 200L, 200); + addDeltaFile(t, null, 201L, 205L, 5); + addDeltaFile(t, null, 206L, 211L, 6); burnThroughTransactions(210); @@ -489,7 +469,7 @@ public void noCompactTableNotEnoughDeltas() throws Exception { LockResponse res = txnHandler.lock(req); txnHandler.commitTxn(new CommitTxnRequest(txnid)); - startInitiator(conf); + startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); Assert.assertEquals(0, rsp.getCompactsSize()); @@ -499,20 +479,18 @@ public void noCompactTableNotEnoughDeltas() throws Exception { public void chooseMajorOverMinorWhenBothValid() throws Exception { Table t = newTable("default", "cmomwbv", false); - HiveConf conf = new HiveConf(); - - addBaseFile(conf, t, null, 200L, 200); - addDeltaFile(conf, t, null, 201L, 211L, 11); - addDeltaFile(conf, t, null, 212L, 222L, 11); - addDeltaFile(conf, t, null, 223L, 233L, 11); - addDeltaFile(conf, t, null, 234L, 244L, 11); - addDeltaFile(conf, t, null, 245L, 255L, 11); - addDeltaFile(conf, t, null, 256L, 266L, 11); - addDeltaFile(conf, t, null, 267L, 277L, 11); - addDeltaFile(conf, t, null, 278L, 288L, 11); - addDeltaFile(conf, t, null, 289L, 299L, 11); - addDeltaFile(conf, t, null, 300L, 310L, 11); - addDeltaFile(conf, t, null, 311L, 321L, 11); + addBaseFile(t, null, 200L, 200); + addDeltaFile(t, null, 201L, 211L, 11); + addDeltaFile(t, null, 212L, 222L, 11); + addDeltaFile(t, null, 223L, 233L, 11); + addDeltaFile(t, null, 234L, 244L, 11); + addDeltaFile(t, null, 245L, 255L, 11); + addDeltaFile(t, null, 256L, 266L, 11); + addDeltaFile(t, null, 267L, 277L, 11); + addDeltaFile(t, null, 278L, 288L, 11); + addDeltaFile(t, null, 289L, 299L, 11); + addDeltaFile(t, null, 300L, 310L, 11); + addDeltaFile(t, null, 311L, 321L, 11); burnThroughTransactions(320); @@ -526,7 +504,7 @@ public void chooseMajorOverMinorWhenBothValid() throws Exception { LockResponse res = txnHandler.lock(req); txnHandler.commitTxn(new CommitTxnRequest(txnid)); - startInitiator(conf); + startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List compacts = rsp.getCompacts(); @@ -541,19 +519,17 @@ public void enoughDeltasNoBase() throws Exception { Table t = newTable("default", "ednb", true); Partition p = newPartition(t, "today"); - HiveConf conf = new HiveConf(); - - addDeltaFile(conf, t, p, 1L, 201L, 200); - addDeltaFile(conf, t, p, 202L, 202L, 1); - addDeltaFile(conf, t, p, 203L, 203L, 1); - addDeltaFile(conf, t, p, 204L, 204L, 1); - addDeltaFile(conf, t, p, 205L, 205L, 1); - addDeltaFile(conf, t, p, 206L, 206L, 1); - addDeltaFile(conf, t, p, 207L, 207L, 1); - addDeltaFile(conf, t, p, 208L, 208L, 1); - addDeltaFile(conf, t, p, 209L, 209L, 1); - addDeltaFile(conf, t, p, 210L, 210L, 1); - addDeltaFile(conf, t, p, 211L, 211L, 1); + addDeltaFile(t, p, 1L, 201L, 200); + addDeltaFile(t, p, 202L, 202L, 1); + addDeltaFile(t, p, 203L, 203L, 1); + addDeltaFile(t, p, 204L, 204L, 1); + addDeltaFile(t, p, 205L, 205L, 1); + addDeltaFile(t, p, 206L, 206L, 1); + addDeltaFile(t, p, 207L, 207L, 1); + addDeltaFile(t, p, 208L, 208L, 1); + addDeltaFile(t, p, 209L, 209L, 1); + addDeltaFile(t, p, 210L, 210L, 1); + addDeltaFile(t, p, 211L, 211L, 1); burnThroughTransactions(210); @@ -568,7 +544,7 @@ public void enoughDeltasNoBase() throws Exception { LockResponse res = txnHandler.lock(req); txnHandler.commitTxn(new CommitTxnRequest(txnid)); - startInitiator(conf); + startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List compacts = rsp.getCompacts(); @@ -584,11 +560,9 @@ public void twoTxnsOnSamePartitionGenerateOneCompactionRequest() throws Exceptio Table t = newTable("default", "ttospgocr", true); Partition p = newPartition(t, "today"); - HiveConf conf = new HiveConf(); - - addBaseFile(conf, t, p, 20L, 20); - addDeltaFile(conf, t, p, 21L, 22L, 2); - addDeltaFile(conf, t, p, 23L, 24L, 2); + addBaseFile(t, p, 20L, 20); + addDeltaFile(t, p, 21L, 22L, 2); + addDeltaFile(t, p, 23L, 24L, 2); burnThroughTransactions(23); @@ -614,7 +588,7 @@ public void twoTxnsOnSamePartitionGenerateOneCompactionRequest() throws Exceptio res = txnHandler.lock(req); txnHandler.commitTxn(new CommitTxnRequest(txnid)); - startInitiator(conf); + startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List compacts = rsp.getCompacts(); @@ -626,9 +600,4 @@ public void twoTxnsOnSamePartitionGenerateOneCompactionRequest() throws Exceptio } // TODO test compactions with legacy file types - - @Before - public void setUpTxnDb() throws Exception { - TxnDbUtil.setConfValues(new HiveConf()); - } } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestWorker.java b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestWorker.java index 90a722b..e525ef9 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestWorker.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestWorker.java @@ -48,7 +48,7 @@ public TestWorker() throws Exception { public void nothing() throws Exception { // Test that the whole things works when there's nothing in the queue. This is just a // survival test. - startWorker(new HiveConf()); + startWorker(); } @Test @@ -205,19 +205,17 @@ public void sortedTable() throws Exception { Table t = newTable("default", "st", false, new HashMap(), sortCols); - HiveConf conf = new HiveConf(); - - addBaseFile(conf, t, null, 20L, 20); - addDeltaFile(conf, t, null, 21L, 22L, 2); - addDeltaFile(conf, t, null, 23L, 24L, 2); - addDeltaFile(conf, t, null, 21L, 24L, 4); + addBaseFile(t, null, 20L, 20); + addDeltaFile(t, null, 21L, 22L, 2); + addDeltaFile(t, null, 23L, 24L, 2); + addDeltaFile(t, null, 21L, 24L, 4); burnThroughTransactions(25); CompactionRequest rqst = new CompactionRequest("default", "st", CompactionType.MINOR); txnHandler.compact(rqst); - startWorker(new HiveConf()); + startWorker(); // There should still be four directories in the location. FileSystem fs = FileSystem.get(conf); @@ -232,12 +230,11 @@ public void sortedPartition() throws Exception { Table t = newTable("default", "sp", true, new HashMap(), sortCols); Partition p = newPartition(t, "today", sortCols); - HiveConf conf = new HiveConf(); - addBaseFile(conf, t, p, 20L, 20); - addDeltaFile(conf, t, p, 21L, 22L, 2); - addDeltaFile(conf, t, p, 23L, 24L, 2); - addDeltaFile(conf, t, p, 21L, 24L, 4); + addBaseFile(t, p, 20L, 20); + addDeltaFile(t, p, 21L, 22L, 2); + addDeltaFile(t, p, 23L, 24L, 2); + addDeltaFile(t, p, 21L, 24L, 4); burnThroughTransactions(25); @@ -245,7 +242,7 @@ public void sortedPartition() throws Exception { rqst.setPartitionname("ds=today"); txnHandler.compact(rqst); - startWorker(new HiveConf()); + startWorker(); // There should still be four directories in the location. FileSystem fs = FileSystem.get(conf); @@ -258,18 +255,16 @@ public void minorTableWithBase() throws Exception { LOG.debug("Starting minorTableWithBase"); Table t = newTable("default", "mtwb", false); - HiveConf conf = new HiveConf(); - - addBaseFile(conf, t, null, 20L, 20); - addDeltaFile(conf, t, null, 21L, 22L, 2); - addDeltaFile(conf, t, null, 23L, 24L, 2); + addBaseFile(t, null, 20L, 20); + addDeltaFile(t, null, 21L, 22L, 2); + addDeltaFile(t, null, 23L, 24L, 2); burnThroughTransactions(25); CompactionRequest rqst = new CompactionRequest("default", "mtwb", CompactionType.MINOR); txnHandler.compact(rqst); - startWorker(conf); + startWorker(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List compacts = rsp.getCompacts(); @@ -304,11 +299,10 @@ public void minorTableWithBase() throws Exception { public void minorPartitionWithBase() throws Exception { Table t = newTable("default", "mpwb", true); Partition p = newPartition(t, "today"); - HiveConf conf = new HiveConf(); - addBaseFile(conf, t, p, 20L, 20); - addDeltaFile(conf, t, p, 21L, 22L, 2); - addDeltaFile(conf, t, p, 23L, 24L, 2); + addBaseFile(t, p, 20L, 20); + addDeltaFile(t, p, 21L, 22L, 2); + addDeltaFile(t, p, 23L, 24L, 2); burnThroughTransactions(25); @@ -316,7 +310,7 @@ public void minorPartitionWithBase() throws Exception { rqst.setPartitionname("ds=today"); txnHandler.compact(rqst); - startWorker(new HiveConf()); + startWorker(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List compacts = rsp.getCompacts(); @@ -351,17 +345,15 @@ public void minorTableNoBase() throws Exception { LOG.debug("Starting minorTableWithBase"); Table t = newTable("default", "mtnb", false); - HiveConf conf = new HiveConf(); - - addDeltaFile(conf, t, null, 1L, 2L, 2); - addDeltaFile(conf, t, null, 3L, 4L, 2); + addDeltaFile(t, null, 1L, 2L, 2); + addDeltaFile(t, null, 3L, 4L, 2); burnThroughTransactions(5); CompactionRequest rqst = new CompactionRequest("default", "mtnb", CompactionType.MINOR); txnHandler.compact(rqst); - startWorker(new HiveConf()); + startWorker(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List compacts = rsp.getCompacts(); @@ -396,18 +388,16 @@ public void majorTableWithBase() throws Exception { LOG.debug("Starting majorTableWithBase"); Table t = newTable("default", "matwb", false); - HiveConf conf = new HiveConf(); - - addBaseFile(conf, t, null, 20L, 20); - addDeltaFile(conf, t, null, 21L, 22L, 2); - addDeltaFile(conf, t, null, 23L, 24L, 2); + addBaseFile(t, null, 20L, 20); + addDeltaFile(t, null, 21L, 22L, 2); + addDeltaFile(t, null, 23L, 24L, 2); burnThroughTransactions(25); CompactionRequest rqst = new CompactionRequest("default", "matwb", CompactionType.MAJOR); txnHandler.compact(rqst); - startWorker(new HiveConf()); + startWorker(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List compacts = rsp.getCompacts(); @@ -428,8 +418,8 @@ public void majorTableWithBase() throws Exception { Assert.assertEquals(2, buckets.length); Assert.assertTrue(buckets[0].getPath().getName().matches("bucket_0000[01]")); Assert.assertTrue(buckets[1].getPath().getName().matches("bucket_0000[01]")); - Assert.assertEquals(1248L, buckets[0].getLen()); - Assert.assertEquals(1248L, buckets[1].getLen()); + Assert.assertEquals(624L, buckets[0].getLen()); + Assert.assertEquals(624L, buckets[1].getLen()); } else { LOG.debug("This is not the file you are looking for " + stat[i].getPath().getName()); } @@ -442,11 +432,10 @@ public void majorPartitionWithBase() throws Exception { LOG.debug("Starting majorPartitionWithBase"); Table t = newTable("default", "mapwb", true); Partition p = newPartition(t, "today"); - HiveConf conf = new HiveConf(); - addBaseFile(conf, t, p, 20L, 20); - addDeltaFile(conf, t, p, 21L, 22L, 2); - addDeltaFile(conf, t, p, 23L, 24L, 2); + addBaseFile(t, p, 20L, 20); + addDeltaFile(t, p, 21L, 22L, 2); + addDeltaFile(t, p, 23L, 24L, 2); burnThroughTransactions(25); @@ -454,7 +443,7 @@ public void majorPartitionWithBase() throws Exception { rqst.setPartitionname("ds=today"); txnHandler.compact(rqst); - startWorker(new HiveConf()); + startWorker(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List compacts = rsp.getCompacts(); @@ -475,8 +464,8 @@ public void majorPartitionWithBase() throws Exception { Assert.assertEquals(2, buckets.length); Assert.assertTrue(buckets[0].getPath().getName().matches("bucket_0000[01]")); Assert.assertTrue(buckets[1].getPath().getName().matches("bucket_0000[01]")); - Assert.assertEquals(1248L, buckets[0].getLen()); - Assert.assertEquals(1248L, buckets[1].getLen()); + Assert.assertEquals(624L, buckets[0].getLen()); + Assert.assertEquals(624L, buckets[1].getLen()); } else { LOG.debug("This is not the file you are looking for " + stat[i].getPath().getName()); } @@ -489,17 +478,15 @@ public void majorTableNoBase() throws Exception { LOG.debug("Starting majorTableNoBase"); Table t = newTable("default", "matnb", false); - HiveConf conf = new HiveConf(); - - addDeltaFile(conf, t, null, 1L, 2L, 2); - addDeltaFile(conf, t, null, 3L, 4L, 2); + addDeltaFile(t, null, 1L, 2L, 2); + addDeltaFile(t, null, 3L, 4L, 2); burnThroughTransactions(5); CompactionRequest rqst = new CompactionRequest("default", "matnb", CompactionType.MAJOR); txnHandler.compact(rqst); - startWorker(new HiveConf()); + startWorker(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List compacts = rsp.getCompacts(); @@ -520,8 +507,8 @@ public void majorTableNoBase() throws Exception { Assert.assertEquals(2, buckets.length); Assert.assertTrue(buckets[0].getPath().getName().matches("bucket_0000[01]")); Assert.assertTrue(buckets[1].getPath().getName().matches("bucket_0000[01]")); - Assert.assertEquals(208L, buckets[0].getLen()); - Assert.assertEquals(208L, buckets[1].getLen()); + Assert.assertEquals(104L, buckets[0].getLen()); + Assert.assertEquals(104L, buckets[1].getLen()); } else { LOG.debug("This is not the file you are looking for " + stat[i].getPath().getName()); } @@ -534,18 +521,16 @@ public void majorTableLegacy() throws Exception { LOG.debug("Starting majorTableLegacy"); Table t = newTable("default", "matl", false); - HiveConf conf = new HiveConf(); - - addLegacyFile(conf, t, null, 20); - addDeltaFile(conf, t, null, 21L, 22L, 2); - addDeltaFile(conf, t, null, 23L, 24L, 2); + addLegacyFile(t, null, 20); + addDeltaFile(t, null, 21L, 22L, 2); + addDeltaFile(t, null, 23L, 24L, 2); burnThroughTransactions(25); CompactionRequest rqst = new CompactionRequest("default", "matl", CompactionType.MAJOR); txnHandler.compact(rqst); - startWorker(new HiveConf()); + startWorker(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List compacts = rsp.getCompacts(); @@ -566,8 +551,8 @@ public void majorTableLegacy() throws Exception { Assert.assertEquals(2, buckets.length); Assert.assertTrue(buckets[0].getPath().getName().matches("bucket_0000[01]")); Assert.assertTrue(buckets[1].getPath().getName().matches("bucket_0000[01]")); - Assert.assertEquals(1248L, buckets[0].getLen()); - Assert.assertEquals(1248L, buckets[1].getLen()); + Assert.assertEquals(624L, buckets[0].getLen()); + Assert.assertEquals(624L, buckets[1].getLen()); } else { LOG.debug("This is not the file you are looking for " + stat[i].getPath().getName()); } @@ -580,18 +565,16 @@ public void minorTableLegacy() throws Exception { LOG.debug("Starting minorTableLegacy"); Table t = newTable("default", "mtl", false); - HiveConf conf = new HiveConf(); - - addLegacyFile(conf, t, null, 20); - addDeltaFile(conf, t, null, 21L, 22L, 2); - addDeltaFile(conf, t, null, 23L, 24L, 2); + addLegacyFile(t, null, 20); + addDeltaFile(t, null, 21L, 22L, 2); + addDeltaFile(t, null, 23L, 24L, 2); burnThroughTransactions(25); CompactionRequest rqst = new CompactionRequest("default", "mtl", CompactionType.MINOR); txnHandler.compact(rqst); - startWorker(new HiveConf()); + startWorker(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List compacts = rsp.getCompacts(); @@ -622,11 +605,11 @@ public void minorTableLegacy() throws Exception { public void majorPartitionWithBaseMissingBuckets() throws Exception { Table t = newTable("default", "mapwbmb", true); Partition p = newPartition(t, "today"); - HiveConf conf = new HiveConf(); - addBaseFile(conf, t, p, 20L, 20, 2, false); - addDeltaFile(conf, t, p, 21L, 22L, 2, 2, false); - addDeltaFile(conf, t, p, 23L, 24L, 2); + + addBaseFile(t, p, 20L, 20, 2, false); + addDeltaFile(t, p, 21L, 22L, 2, 2, false); + addDeltaFile(t, p, 23L, 26L, 4); burnThroughTransactions(25); @@ -634,7 +617,7 @@ public void majorPartitionWithBaseMissingBuckets() throws Exception { rqst.setPartitionname("ds=today"); txnHandler.compact(rqst); - startWorker(new HiveConf()); + startWorker(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List compacts = rsp.getCompacts(); @@ -649,7 +632,7 @@ public void majorPartitionWithBaseMissingBuckets() throws Exception { // Find the new delta file and make sure it has the right contents boolean sawNewBase = false; for (int i = 0; i < stat.length; i++) { - if (stat[i].getPath().getName().equals("base_0000024")) { + if (stat[i].getPath().getName().equals("base_0000026")) { sawNewBase = true; FileStatus[] buckets = fs.listStatus(stat[i].getPath()); Assert.assertEquals(2, buckets.length); @@ -658,10 +641,12 @@ public void majorPartitionWithBaseMissingBuckets() throws Exception { // Bucket 0 should be small and bucket 1 should be large, make sure that's the case Assert.assertTrue( ("bucket_00000".equals(buckets[0].getPath().getName()) && 104L == buckets[0].getLen() - && "bucket_00001".equals(buckets[1].getPath().getName()) && 1248L == buckets[1] .getLen()) + && "bucket_00001".equals(buckets[1].getPath().getName()) && 676L == buckets[1] + .getLen()) || ("bucket_00000".equals(buckets[1].getPath().getName()) && 104L == buckets[1].getLen() - && "bucket_00001".equals(buckets[0].getPath().getName()) && 1248L == buckets[0] .getLen()) + && "bucket_00001".equals(buckets[0].getPath().getName()) && 676L == buckets[0] + .getLen()) ); } else { LOG.debug("This is not the file you are looking for " + stat[i].getPath().getName()); @@ -669,9 +654,4 @@ public void majorPartitionWithBaseMissingBuckets() throws Exception { } Assert.assertTrue(sawNewBase); } - - @Before - public void setUpTxnDb() throws Exception { - TxnDbUtil.setConfValues(new HiveConf()); - } } diff --git a/ql/src/test/queries/clientnegative/alter_partition_partial_spec_dyndisabled.q b/ql/src/test/queries/clientnegative/alter_partition_partial_spec_dyndisabled.q new file mode 100644 index 0000000..74bfb43 --- /dev/null +++ b/ql/src/test/queries/clientnegative/alter_partition_partial_spec_dyndisabled.q @@ -0,0 +1,13 @@ +SET hive.exec.dynamic.partition = true; +SET hive.exec.dynamic.partition.mode = nonstrict; + +create table alter_partition_partial_spec_dyndisabled0 (c1 string) partitioned by (p1 string, p2 string); + +alter table alter_partition_partial_spec_dyndisabled0 add partition (p1='abc', p2='123'); +alter table alter_partition_partial_spec_dyndisabled0 partition (p1, p2) change c1 c1 int; + +describe alter_partition_partial_spec_dyndisabled0 partition (p1='abc', p2='123'); + +SET hive.exec.dynamic.partition = false; +-- Same statement should fail if dynamic partitioning disabled +alter table alter_partition_partial_spec_dyndisabled0 partition (p1, p2) change c1 c1 int; diff --git a/ql/src/test/queries/clientnegative/ambiguous_col0.q b/ql/src/test/queries/clientnegative/ambiguous_col0.q index 46349c6..e69de29 100644 --- a/ql/src/test/queries/clientnegative/ambiguous_col0.q +++ b/ql/src/test/queries/clientnegative/ambiguous_col0.q @@ -1,2 +0,0 @@ --- TOK_ALLCOLREF -explain select * from (select * from (select * from src) a join (select * from src1) b on (a.key = b.key)) t; diff --git a/ql/src/test/queries/clientnegative/ambiguous_col1.q b/ql/src/test/queries/clientnegative/ambiguous_col1.q index 9e8bcbd..e69de29 100644 --- a/ql/src/test/queries/clientnegative/ambiguous_col1.q +++ b/ql/src/test/queries/clientnegative/ambiguous_col1.q @@ -1,3 +0,0 @@ -set hive.support.quoted.identifiers=none; --- TOK_TABLE_OR_COL -explain select * from (select `.*` from (select * from src) a join (select * from src1) b on (a.key = b.key)) t; diff --git a/ql/src/test/queries/clientnegative/ambiguous_col2.q b/ql/src/test/queries/clientnegative/ambiguous_col2.q index 33d4aed..e69de29 100644 --- a/ql/src/test/queries/clientnegative/ambiguous_col2.q +++ b/ql/src/test/queries/clientnegative/ambiguous_col2.q @@ -1,3 +0,0 @@ -set hive.support.quoted.identifiers=none; --- DOT -explain select * from (select a.`[kv].*`, b.`[kv].*` from (select * from src) a join (select * from src1) b on (a.key = b.key)) t; diff --git a/ql/src/test/queries/clientnegative/authorization_disallow_transform.q b/ql/src/test/queries/clientnegative/authorization_disallow_transform.q index 342c29a..1702bcf 100644 --- a/ql/src/test/queries/clientnegative/authorization_disallow_transform.q +++ b/ql/src/test/queries/clientnegative/authorization_disallow_transform.q @@ -1,4 +1,6 @@ set hive.test.authz.sstd.hs2.mode=true; set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; +set hive.security.authorization.enabled=true; set role ALL; -SELECT TRANSFORM (*) USING 'cat' AS (key, value) FROM src; +create table t1(i int); +SELECT TRANSFORM (*) USING 'cat' AS (key, value) FROM t1; diff --git a/ql/src/test/queries/clientnegative/authorization_set_invalidconf.q b/ql/src/test/queries/clientnegative/authorization_set_invalidconf.q new file mode 100644 index 0000000..8dbb3e7 --- /dev/null +++ b/ql/src/test/queries/clientnegative/authorization_set_invalidconf.q @@ -0,0 +1,8 @@ +set hive.test.authz.sstd.hs2.mode=true; +set hive.security.authorization.enabled=true; +set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory; + +-- run a sql query to initialize authorization, then try setting a allowed config and then a disallowed config param +use default; +set hive.optimize.listbucketing=true; +set hive.security.authorization.enabled=true; diff --git a/ql/src/test/queries/clientnegative/set_hiveconf_validation2.q b/ql/src/test/queries/clientnegative/set_hiveconf_validation2.q index 579e940..ad14c0c 100644 --- a/ql/src/test/queries/clientnegative/set_hiveconf_validation2.q +++ b/ql/src/test/queries/clientnegative/set_hiveconf_validation2.q @@ -1,4 +1,4 @@ --- should fail: hive.fetch.task.conversion accepts minimal or more +-- should fail: hive.fetch.task.conversion accepts none, minimal or more desc src; set hive.conf.validation=true; diff --git a/ql/src/test/queries/clientpositive/acid_vectorization.q b/ql/src/test/queries/clientpositive/acid_vectorization.q index 3f386c9..4b11412 100644 --- a/ql/src/test/queries/clientpositive/acid_vectorization.q +++ b/ql/src/test/queries/clientpositive/acid_vectorization.q @@ -12,3 +12,5 @@ set hive.vectorized.execution.enabled=true; update acid_vectorized set b = 'foo' where b = 'bar'; set hive.vectorized.execution.enabled=true; delete from acid_vectorized where b = 'foo'; +set hive.vectorized.execution.enabled=true; +select a, b from acid_vectorized order by a, b; diff --git a/ql/src/test/queries/clientpositive/acid_vectorization_partition.q b/ql/src/test/queries/clientpositive/acid_vectorization_partition.q new file mode 100644 index 0000000..9348d05 --- /dev/null +++ b/ql/src/test/queries/clientpositive/acid_vectorization_partition.q @@ -0,0 +1,10 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.enforce.bucketing=true; +set hive.exec.dynamic.partition.mode=nonstrict; + +CREATE TABLE acid_vectorized_part(a INT, b STRING) partitioned by (ds string) CLUSTERED BY(a) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true'); +insert into table acid_vectorized_part partition (ds = 'today') select cint, cstring1 from alltypesorc where cint is not null order by cint limit 10; +insert into table acid_vectorized_part partition (ds = 'tomorrow') select cint, cstring1 from alltypesorc where cint is not null order by cint limit 10; +set hive.vectorized.execution.enabled=true; +select * from acid_vectorized_part order by a, b; diff --git a/ql/src/test/queries/clientpositive/acid_vectorization_project.q b/ql/src/test/queries/clientpositive/acid_vectorization_project.q new file mode 100644 index 0000000..a44b57a --- /dev/null +++ b/ql/src/test/queries/clientpositive/acid_vectorization_project.q @@ -0,0 +1,11 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.enforce.bucketing=true; +set hive.exec.dynamic.partition.mode=nonstrict; + +CREATE TABLE acid_vectorized(a INT, b STRING, c float) CLUSTERED BY(a) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true'); +insert into table acid_vectorized select cint, cstring1, cfloat from alltypesorc where cint is not null order by cint limit 10; +set hive.vectorized.execution.enabled=true; +select a,b from acid_vectorized order by a; +select a,c from acid_vectorized order by a; +select b,c from acid_vectorized order by b; diff --git a/ql/src/test/queries/clientpositive/alter_partition_change_col.q b/ql/src/test/queries/clientpositive/alter_partition_change_col.q index baabb9f..4451ef3 100644 --- a/ql/src/test/queries/clientpositive/alter_partition_change_col.q +++ b/ql/src/test/queries/clientpositive/alter_partition_change_col.q @@ -6,54 +6,70 @@ SET hive.exec.dynamic.partition.mode = nonstrict; create table alter_partition_change_col0 (c1 string, c2 string); load data local inpath '../../data/files/dec.txt' overwrite into table alter_partition_change_col0; -create table alter_partition_change_col1 (c1 string, c2 string) partitioned by (p1 string); +create table alter_partition_change_col1 (c1 string, c2 string) partitioned by (p1 string, p2 string); -insert overwrite table alter_partition_change_col1 partition (p1) - select c1, c2, 'abc' from alter_partition_change_col0 +insert overwrite table alter_partition_change_col1 partition (p1, p2) + select c1, c2, 'abc', '123' from alter_partition_change_col0 union all - select c1, c2, null from alter_partition_change_col0; + select c1, c2, null, '123' from alter_partition_change_col0; show partitions alter_partition_change_col1; -select * from alter_partition_change_col1; +select * from alter_partition_change_col1 where p1='abc'; +select * from alter_partition_change_col1 where p1='__HIVE_DEFAULT_PARTITION__'; -- Change c2 to decimal(10,0) alter table alter_partition_change_col1 change c2 c2 decimal(10,0); -alter table alter_partition_change_col1 partition (p1='abc') change c2 c2 decimal(10,0); -alter table alter_partition_change_col1 partition (p1='__HIVE_DEFAULT_PARTITION__') change c2 c2 decimal(10,0); -select * from alter_partition_change_col1; +alter table alter_partition_change_col1 partition (p1='abc', p2='123') change c2 c2 decimal(10,0); +alter table alter_partition_change_col1 partition (p1='__HIVE_DEFAULT_PARTITION__', p2='123') change c2 c2 decimal(10,0); +select * from alter_partition_change_col1 where p1='abc'; +select * from alter_partition_change_col1 where p1='__HIVE_DEFAULT_PARTITION__'; -- Change the column type at the table level. Table-level describe shows the new type, but the existing partition does not. alter table alter_partition_change_col1 change c2 c2 decimal(14,4); describe alter_partition_change_col1; -describe alter_partition_change_col1 partition (p1='abc'); -select * from alter_partition_change_col1; +describe alter_partition_change_col1 partition (p1='abc', p2='123'); +select * from alter_partition_change_col1 where p1='abc'; +select * from alter_partition_change_col1 where p1='__HIVE_DEFAULT_PARTITION__'; -- now change the column type of the existing partition -alter table alter_partition_change_col1 partition (p1='abc') change c2 c2 decimal(14,4); -describe alter_partition_change_col1 partition (p1='abc'); -select * from alter_partition_change_col1; +alter table alter_partition_change_col1 partition (p1='abc', p2='123') change c2 c2 decimal(14,4); +describe alter_partition_change_col1 partition (p1='abc', p2='123'); +select * from alter_partition_change_col1 where p1='abc'; +select * from alter_partition_change_col1 where p1='__HIVE_DEFAULT_PARTITION__'; -- change column for default partition value -alter table alter_partition_change_col1 partition (p1='__HIVE_DEFAULT_PARTITION__') change c2 c2 decimal(14,4); -describe alter_partition_change_col1 partition (p1='__HIVE_DEFAULT_PARTITION__'); -select * from alter_partition_change_col1; +alter table alter_partition_change_col1 partition (p1='__HIVE_DEFAULT_PARTITION__', p2='123') change c2 c2 decimal(14,4); +describe alter_partition_change_col1 partition (p1='__HIVE_DEFAULT_PARTITION__', p2='123'); +select * from alter_partition_change_col1 where p1='abc'; +select * from alter_partition_change_col1 where p1='__HIVE_DEFAULT_PARTITION__'; -- Try out replace columns -alter table alter_partition_change_col1 partition (p1='abc') replace columns (c1 string); +alter table alter_partition_change_col1 partition (p1='abc', p2='123') replace columns (c1 string); describe alter_partition_change_col1; -describe alter_partition_change_col1 partition (p1='abc'); -select * from alter_partition_change_col1; +describe alter_partition_change_col1 partition (p1='abc', p2='123'); +select * from alter_partition_change_col1 where p1='abc'; +select * from alter_partition_change_col1 where p1='__HIVE_DEFAULT_PARTITION__'; + alter table alter_partition_change_col1 replace columns (c1 string); describe alter_partition_change_col1; -select * from alter_partition_change_col1; +select * from alter_partition_change_col1 where p1='abc'; +select * from alter_partition_change_col1 where p1='__HIVE_DEFAULT_PARTITION__'; -- Try add columns alter table alter_partition_change_col1 add columns (c2 decimal(14,4)); describe alter_partition_change_col1; -describe alter_partition_change_col1 partition (p1='abc'); -select * from alter_partition_change_col1; +describe alter_partition_change_col1 partition (p1='abc', p2='123'); +select * from alter_partition_change_col1 where p1='abc'; +select * from alter_partition_change_col1 where p1='__HIVE_DEFAULT_PARTITION__'; -alter table alter_partition_change_col1 partition (p1='abc') add columns (c2 decimal(14,4)); -describe alter_partition_change_col1 partition (p1='abc'); -select * from alter_partition_change_col1; +alter table alter_partition_change_col1 partition (p1='abc', p2='123') add columns (c2 decimal(14,4)); +describe alter_partition_change_col1 partition (p1='abc', p2='123'); +select * from alter_partition_change_col1 where p1='abc'; +select * from alter_partition_change_col1 where p1='__HIVE_DEFAULT_PARTITION__'; +-- Try changing column for all partitions at once +alter table alter_partition_change_col1 partition (p1, p2='123') change column c2 c2 decimal(10,0); +describe alter_partition_change_col1 partition (p1='abc', p2='123'); +describe alter_partition_change_col1 partition (p1='__HIVE_DEFAULT_PARTITION__', p2='123'); +select * from alter_partition_change_col1 where p1='abc'; +select * from alter_partition_change_col1 where p1='__HIVE_DEFAULT_PARTITION__'; diff --git a/ql/src/test/queries/clientpositive/annotate_stats_join_pkfk.q b/ql/src/test/queries/clientpositive/annotate_stats_join_pkfk.q new file mode 100644 index 0000000..aa62c60 --- /dev/null +++ b/ql/src/test/queries/clientpositive/annotate_stats_join_pkfk.q @@ -0,0 +1,123 @@ +set hive.stats.fetch.column.stats=true; + +drop table store_sales; +drop table store; +drop table customer_address; + +-- s_store_sk is PK, ss_store_sk is FK +-- ca_address_sk is PK, ss_addr_sk is FK + +create table store_sales +( + ss_sold_date_sk int, + ss_sold_time_sk int, + ss_item_sk int, + ss_customer_sk int, + ss_cdemo_sk int, + ss_hdemo_sk int, + ss_addr_sk int, + ss_store_sk int, + ss_promo_sk int, + ss_ticket_number int, + ss_quantity int, + ss_wholesale_cost float, + ss_list_price float, + ss_sales_price float, + ss_ext_discount_amt float, + ss_ext_sales_price float, + ss_ext_wholesale_cost float, + ss_ext_list_price float, + ss_ext_tax float, + ss_coupon_amt float, + ss_net_paid float, + ss_net_paid_inc_tax float, + ss_net_profit float +) +row format delimited fields terminated by '|'; + +create table store +( + s_store_sk int, + s_store_id string, + s_rec_start_date string, + s_rec_end_date string, + s_closed_date_sk int, + s_store_name string, + s_number_employees int, + s_floor_space int, + s_hours string, + s_manager string, + s_market_id int, + s_geography_class string, + s_market_desc string, + s_market_manager string, + s_division_id int, + s_division_name string, + s_company_id int, + s_company_name string, + s_street_number string, + s_street_name string, + s_street_type string, + s_suite_number string, + s_city string, + s_county string, + s_state string, + s_zip string, + s_country string, + s_gmt_offset float, + s_tax_precentage float +) +row format delimited fields terminated by '|'; + +create table customer_address +( + ca_address_sk int, + ca_address_id string, + ca_street_number string, + ca_street_name string, + ca_street_type string, + ca_suite_number string, + ca_city string, + ca_county string, + ca_state string, + ca_zip string, + ca_country string, + ca_gmt_offset float, + ca_location_type string +) +row format delimited fields terminated by '|'; + +load data local inpath '../../data/files/store.txt' overwrite into table store; +load data local inpath '../../data/files/store_sales.txt' overwrite into table store_sales; +load data local inpath '../../data/files/customer_address.txt' overwrite into table customer_address; + +analyze table store compute statistics; +analyze table store compute statistics for columns s_store_sk, s_floor_space; +analyze table store_sales compute statistics; +analyze table store_sales compute statistics for columns ss_store_sk, ss_addr_sk, ss_quantity; +analyze table customer_address compute statistics; +analyze table customer_address compute statistics for columns ca_address_sk; + +explain select s.s_store_sk from store s join store_sales ss on (s.s_store_sk = ss.ss_store_sk); + +explain select s.s_store_sk from store s join store_sales ss on (s.s_store_sk = ss.ss_store_sk) where s.s_store_sk > 0; + +explain select s.s_store_sk from store s join store_sales ss on (s.s_store_sk = ss.ss_store_sk) where s.s_company_id > 0 and ss.ss_quantity > 10; + +explain select s.s_store_sk from store s join store_sales ss on (s.s_store_sk = ss.ss_store_sk) where s.s_floor_space > 0; + +explain select s.s_store_sk from store s join store_sales ss on (s.s_store_sk = ss.ss_store_sk) where ss.ss_quantity > 10; + +explain select s.s_store_sk from store s join store_sales ss on (s.s_store_sk = ss.ss_store_sk) join store s1 on (s1.s_store_sk = ss.ss_store_sk); + +explain select s.s_store_sk from store s join store_sales ss on (s.s_store_sk = ss.ss_store_sk) join store s1 on (s1.s_store_sk = ss.ss_store_sk) where s.s_store_sk > 1000; + +explain select s.s_store_sk from store s join store_sales ss on (s.s_store_sk = ss.ss_store_sk) join store s1 on (s1.s_store_sk = ss.ss_store_sk) where s.s_floor_space > 1000; + +explain select s.s_store_sk from store s join store_sales ss on (s.s_store_sk = ss.ss_store_sk) join store s1 on (s1.s_store_sk = ss.ss_store_sk) where ss.ss_quantity > 10; + +explain select s.s_store_sk from store s join store_sales ss on (s.s_store_sk = ss.ss_store_sk) join customer_address ca on (ca.ca_address_sk = ss.ss_addr_sk); + +drop table store_sales; +drop table store; +drop table customer_address; diff --git a/ql/src/test/queries/clientpositive/authorization_role_grant2.q b/ql/src/test/queries/clientpositive/authorization_role_grant2.q index 59359a7..224ee28 100644 --- a/ql/src/test/queries/clientpositive/authorization_role_grant2.q +++ b/ql/src/test/queries/clientpositive/authorization_role_grant2.q @@ -20,6 +20,7 @@ show principals src_role_wadmin; set user.name=user2; set role src_role_WadMin; +show principals src_role_wadmin; -- grant role to another user grant src_Role_wadmin to user user3; diff --git a/ql/src/test/queries/clientpositive/avro_date.q b/ql/src/test/queries/clientpositive/avro_date.q new file mode 100644 index 0000000..996fa47 --- /dev/null +++ b/ql/src/test/queries/clientpositive/avro_date.q @@ -0,0 +1,24 @@ +DROP TABLE avro_date_staging; +DROP TABLE avro_date; +DROP TABLE avro_date_casts; + +CREATE TABLE avro_date_staging (d date, m1 map, l1 array) + ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' + COLLECTION ITEMS TERMINATED BY ',' MAP KEYS TERMINATED BY ':' + STORED AS TEXTFILE; + +LOAD DATA LOCAL INPATH '../../data/files/avro_date.txt' OVERWRITE INTO TABLE avro_date_staging; + +CREATE TABLE avro_date (d date, m1 map, l1 array) + PARTITIONED BY (p1 int, p2 date) + ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' + COLLECTION ITEMS TERMINATED BY ',' MAP KEYS TERMINATED BY ':' + STORED AS AVRO; + +INSERT OVERWRITE TABLE avro_date PARTITION(p1=2, p2='2014-09-26') SELECT * FROM avro_date_staging; + +SELECT * FROM avro_date; +SELECT d, COUNT(d) FROM avro_date GROUP BY d; +SELECT * FROM avro_date WHERE d!='1947-02-11'; +SELECT * FROM avro_date WHERE d<'2014-12-21'; +SELECT * FROM avro_date WHERE d>'8000-12-01'; diff --git a/ql/src/test/queries/clientpositive/avro_deserialize_map_null.q b/ql/src/test/queries/clientpositive/avro_deserialize_map_null.q new file mode 100644 index 0000000..fa170db --- /dev/null +++ b/ql/src/test/queries/clientpositive/avro_deserialize_map_null.q @@ -0,0 +1,14 @@ +-- These test attempts to deserialize an Avro file that contains map null values, and the file schema +-- vs record schema have the null values in different positions +-- i.e. +-- fileSchema = [{ "type" : "map", "values" : ["string","null"]}, "null"] +-- recordSchema = ["null", { "type" : "map", "values" : ["string","null"]}] + + +DROP TABLE IF EXISTS avro_table; + +CREATE TABLE avro_table (avreau_col_1 map) STORED AS AVRO; +LOAD DATA LOCAL INPATH '../../data/files/map_null_val.avro' OVERWRITE INTO TABLE avro_table; +SELECT * FROM avro_table; + +DROP TABLE avro_table; \ No newline at end of file diff --git a/ql/src/test/queries/clientpositive/avro_schema_error_message.q b/ql/src/test/queries/clientpositive/avro_schema_error_message.q index cf1fda1..e69de29 100644 --- a/ql/src/test/queries/clientpositive/avro_schema_error_message.q +++ b/ql/src/test/queries/clientpositive/avro_schema_error_message.q @@ -1,11 +0,0 @@ --- verify we get the sentinel schema if we don't provide one - -CREATE TABLE avro_with_no_schema -ROW FORMAT -SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' -STORED AS -INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' -OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat'; - -DESCRIBE avro_with_no_schema; - diff --git a/ql/src/test/queries/clientpositive/cast_qualified_types.q b/ql/src/test/queries/clientpositive/cast_qualified_types.q new file mode 100644 index 0000000..fe0abd4 --- /dev/null +++ b/ql/src/test/queries/clientpositive/cast_qualified_types.q @@ -0,0 +1,9 @@ +set hive.plan.serialization.format=javaXML; + +select + cast(key as decimal(10,2)) as c1, + cast(key as char(10)) as c2, + cast(key as varchar(10)) as c3 +from src +order by c1, c2, c3 +limit 1; diff --git a/ql/src/test/queries/clientpositive/cbo_correctness.q b/ql/src/test/queries/clientpositive/cbo_correctness.q index f7f0722..bb328f6 100644 --- a/ql/src/test/queries/clientpositive/cbo_correctness.q +++ b/ql/src/test/queries/clientpositive/cbo_correctness.q @@ -251,7 +251,7 @@ drop view v3; drop view v4; -- 11. Union All -select * from t1 union all select * from t2 order by key; +select * from t1 union all select * from t2 order by key, c_boolean, value, dt; select key from (select key, c_int from (select * from t1 union all select * from t2 where t2.key >=0)r1 union all select key, c_int from t3)r2 where key >=0 order by key; select r2.key from (select key, c_int from (select key, c_int from t1 union all select key, c_int from t3 )r1 union all select key, c_int from t3)r2 join (select key, c_int from (select * from t1 union all select * from t2 where t2.key >=0)r1 union all select key, c_int from t3)r3 on r2.key=r3.key where r3.key >=0 order by r2.key; @@ -281,7 +281,7 @@ from src_cbo where src_cbo.key not in ( select key from src_cbo s1 where s1.key > '2' - ) + ) order by key ; -- non agg, corr @@ -456,7 +456,33 @@ from (select b.key, count(*) ) a ; --- 17. get stats with empty partition list +-- 20. Test get stats with empty partition list select t1.value from t1 join t2 on t1.key = t2.key where t1.dt = '10' and t1.c_boolean = true; - +-- 21. Test groupby is empty and there is no other cols in aggr +select unionsrc.key FROM (select 'tst1' as key, count(1) as value from src) unionsrc; + +select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src) unionsrc; + +select unionsrc.key FROM (select 'max' as key, max(c_int) as value from t3 s1 + UNION ALL + select 'min' as key, min(c_int) as value from t3 s2 + UNION ALL + select 'avg' as key, avg(c_int) as value from t3 s3) unionsrc order by unionsrc.key; + +select unionsrc.key, unionsrc.value FROM (select 'max' as key, max(c_int) as value from t3 s1 + UNION ALL + select 'min' as key, min(c_int) as value from t3 s2 + UNION ALL + select 'avg' as key, avg(c_int) as value from t3 s3) unionsrc order by unionsrc.key; + +select unionsrc.key, count(1) FROM (select 'max' as key, max(c_int) as value from t3 s1 + UNION ALL + select 'min' as key, min(c_int) as value from t3 s2 + UNION ALL + select 'avg' as key, avg(c_int) as value from t3 s3) unionsrc group by unionsrc.key order by unionsrc.key; + +-- Windowing +select *, rank() over(partition by key order by value) as rr from src1; + +select *, rank() over(partition by key order by value) from src1; diff --git a/ql/src/test/queries/clientpositive/complex_alias.q b/ql/src/test/queries/clientpositive/complex_alias.q new file mode 100644 index 0000000..e2810c3 --- /dev/null +++ b/ql/src/test/queries/clientpositive/complex_alias.q @@ -0,0 +1,46 @@ +CREATE TABLE agg1 (col0 INT, col1 STRING, col2 DOUBLE); + +INSERT INTO TABLE agg1 select key,value,key from src tablesample (1 rows); + +EXPLAIN +SELECT single_use_subq11.a1 AS a1, + single_use_subq11.a2 AS a2 +FROM (SELECT Sum(agg1.col2) AS a1 + FROM agg1 + GROUP BY agg1.col0) single_use_subq12 + JOIN (SELECT alias.a2 AS a0, + alias.a1 AS a1, + alias.a1 AS a2 + FROM (SELECT agg1.col1 AS a0, + '42' AS a1, + agg1.col0 AS a2 + FROM agg1 + UNION ALL + SELECT agg1.col1 AS a0, + '41' AS a1, + agg1.col0 AS a2 + FROM agg1) alias + GROUP BY alias.a2, + alias.a1) single_use_subq11 + ON ( single_use_subq11.a0 = single_use_subq11.a0 ); + +SELECT single_use_subq11.a1 AS a1, + single_use_subq11.a2 AS a2 +FROM (SELECT Sum(agg1.col2) AS a1 + FROM agg1 + GROUP BY agg1.col0) single_use_subq12 + JOIN (SELECT alias.a2 AS a0, + alias.a1 AS a1, + alias.a1 AS a2 + FROM (SELECT agg1.col1 AS a0, + '42' AS a1, + agg1.col0 AS a2 + FROM agg1 + UNION ALL + SELECT agg1.col1 AS a0, + '41' AS a1, + agg1.col0 AS a2 + FROM agg1) alias + GROUP BY alias.a2, + alias.a1) single_use_subq11 + ON ( single_use_subq11.a0 = single_use_subq11.a0 ); diff --git a/ql/src/test/queries/clientpositive/constprog2.q b/ql/src/test/queries/clientpositive/constprog2.q index 72ce5a3..6001668 100644 --- a/ql/src/test/queries/clientpositive/constprog2.q +++ b/ql/src/test/queries/clientpositive/constprog2.q @@ -7,4 +7,10 @@ SELECT src1.key, src1.key + 1, src2.value SELECT src1.key, src1.key + 1, src2.value FROM src src1 join src src2 ON src1.key = src2.key AND src1.key = 86; +EXPLAIN +SELECT src1.key, src1.key + 1, src2.value + FROM src src1 join src src2 ON src1.key = src2.key AND cast(src1.key as double) = 86; + +SELECT src1.key, src1.key + 1, src2.value + FROM src src1 join src src2 ON src1.key = src2.key AND cast(src1.key as double) = 86; diff --git a/ql/src/test/queries/clientpositive/constprog_type.q b/ql/src/test/queries/clientpositive/constprog_type.q index 93249ad..5683a48 100644 --- a/ql/src/test/queries/clientpositive/constprog_type.q +++ b/ql/src/test/queries/clientpositive/constprog_type.q @@ -12,3 +12,28 @@ SELECT cast('2013-11-17' as date), cast(cast('1.3041352164485E9' as double) as t FROM src tablesample (1 rows); SELECT * FROM dest1; + +SELECT key, value FROM src WHERE key = cast(86 as double); + +CREATE TABLE primitives1 ( + id INT , + bool_col BOOLEAN , + tinyint_col TINYINT , + smallint_col SMALLINT , + int_col INT , + bigint_col BIGINT , + float_col FLOAT , + double_col DOUBLE , + date_string_col STRING , + string_col STRING , + timestamp_col TIMESTAMP ) +ROW FORMAT DELIMITED + FIELDS TERMINATED BY ',' + ESCAPED BY '\\' +STORED AS TEXTFILE; + +LOAD DATA LOCAL INPATH '../../data/files/types/primitives/090101.txt' +OVERWRITE INTO TABLE primitives1 ; + + +select id,bool_col,tinyint_col,smallint_col,int_col,bigint_col,float_col,double_col from primitives1 where id = cast (0 as float) and bool_col = cast('true' as boolean) and tinyint_col = cast(0 as double) and smallint_col = cast(0 as bigint) and int_col = cast (0 as double) and bigint_col = cast(0 as tinyint) and float_col = cast(0.0 as string) and double_col = cast (0.0 as float); diff --git a/ql/src/test/queries/clientpositive/create_like.q b/ql/src/test/queries/clientpositive/create_like.q index 13539a6..eb9f191 100644 --- a/ql/src/test/queries/clientpositive/create_like.q +++ b/ql/src/test/queries/clientpositive/create_like.q @@ -33,3 +33,32 @@ DROP TABLE table4; CREATE EXTERNAL TABLE table4 (a INT) LOCATION '${system:hive.root}/data/files/ext_test'; SELECT * FROM table4; + +CREATE TABLE doctors STORED AS AVRO TBLPROPERTIES ('avro.schema.literal'='{ + "namespace": "testing.hive.avro.serde", + "name": "doctors", + "type": "record", + "fields": [ + { + "name":"number", + "type":"int", + "doc":"Order of playing the role" + }, + { + "name":"first_name", + "type":"string", + "doc":"first name of actor playing role" + }, + { + "name":"last_name", + "type":"string", + "doc":"last name of actor playing role" + } + ] +}'); + +alter table doctors set tblproperties ('k1'='v1', 'k2'='v2'); +DESCRIBE FORMATTED doctors; + +CREATE TABLE doctors2 like doctors; +DESCRIBE FORMATTED doctors2; diff --git a/ql/src/test/queries/clientpositive/ctas_colname.q b/ql/src/test/queries/clientpositive/ctas_colname.q index 5322626..890971e 100644 --- a/ql/src/test/queries/clientpositive/ctas_colname.q +++ b/ql/src/test/queries/clientpositive/ctas_colname.q @@ -3,9 +3,11 @@ -- HIVE-4392, column aliases from expressionRR (GBY, etc.) are not valid name for table -- group by + + explain -create table summary as select *, sum(key), count(value) from src; -create table summary as select *, sum(key), count(value) from src; +create table summary as select *, key + 1, concat(value, value) from src limit 20; +create table summary as select *, key + 1, concat(value, value) from src limit 20; describe formatted summary; select * from summary; diff --git a/ql/src/test/queries/clientpositive/decimal_serde.q b/ql/src/test/queries/clientpositive/decimal_serde.q index cf3a86c..be7a4ac 100644 --- a/ql/src/test/queries/clientpositive/decimal_serde.q +++ b/ql/src/test/queries/clientpositive/decimal_serde.q @@ -15,12 +15,15 @@ SELECT * FROM DECIMAL_TEXT ORDER BY key, value; CREATE TABLE DECIMAL_RC STORED AS RCFile AS SELECT * FROM DECIMAL_TEXT; +describe formatted DECIMAL_RC; CREATE TABLE DECIMAL_LAZY_COL ROW FORMAT SERDE "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe" STORED AS RCFile AS SELECT * FROM DECIMAL_RC; +describe formatted DECIMAL_LAZY_COL; + CREATE TABLE DECIMAL_SEQUENCE ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' diff --git a/ql/src/test/queries/clientpositive/drop_index.q b/ql/src/test/queries/clientpositive/drop_index.q index 54ef823..e03856c 100644 --- a/ql/src/test/queries/clientpositive/drop_index.q +++ b/ql/src/test/queries/clientpositive/drop_index.q @@ -1,3 +1,2 @@ -SET hive.exec.drop.ignorenonexistent=false; DROP INDEX IF EXISTS UnknownIndex ON src; DROP INDEX IF EXISTS UnknownIndex ON UnknownTable; diff --git a/ql/src/test/queries/clientpositive/index_auto_partitioned.q b/ql/src/test/queries/clientpositive/index_auto_partitioned.q index 49df8f2..f9f9a1b 100644 --- a/ql/src/test/queries/clientpositive/index_auto_partitioned.q +++ b/ql/src/test/queries/clientpositive/index_auto_partitioned.q @@ -1,4 +1,5 @@ set hive.stats.dbclass=fs; +set hive.fetch.task.conversion=none; -- SORT_QUERY_RESULTS -- test automatic use of index on table with partitions diff --git a/ql/src/test/queries/clientpositive/index_bitmap_auto_partitioned.q b/ql/src/test/queries/clientpositive/index_bitmap_auto_partitioned.q index 011d77a..2c0b676 100644 --- a/ql/src/test/queries/clientpositive/index_bitmap_auto_partitioned.q +++ b/ql/src/test/queries/clientpositive/index_bitmap_auto_partitioned.q @@ -1,4 +1,5 @@ set hive.stats.dbclass=fs; +set hive.fetch.task.conversion=none; -- SORT_QUERY_RESULTS diff --git a/ql/src/test/queries/clientpositive/index_in_db.q b/ql/src/test/queries/clientpositive/index_in_db.q new file mode 100644 index 0000000..253870c --- /dev/null +++ b/ql/src/test/queries/clientpositive/index_in_db.q @@ -0,0 +1,13 @@ +set hive.optimize.index.filter=true; +drop database if exists index_test_db cascade; +-- Test selecting selecting from a table that is backed by an index +-- create table, index in a db, then set default db as current db, and try selecting + +create database index_test_db; + +use index_test_db; +create table testtb (id int, name string); +create index id_index on table testtb (id) as 'COMPACT' WITH DEFERRED REBUILD in table testdb_id_idx_tb; + +use default; +select * from index_test_db.testtb where id>2; diff --git a/ql/src/test/queries/clientpositive/insert0.q b/ql/src/test/queries/clientpositive/insert0.q new file mode 100644 index 0000000..36d01b6 --- /dev/null +++ b/ql/src/test/queries/clientpositive/insert0.q @@ -0,0 +1,38 @@ +set hive.cbo.enable=true; + +DROP TABLE insert_into1; +DROP TABLE ctas_table; +DROP TABLE ctas_part; + +CREATE TABLE insert_into1 (key int, value string); + +INSERT OVERWRITE TABLE insert_into1 SELECT * from src ORDER BY key LIMIT 10; + +select * from insert_into1 order by key; + +INSERT INTO TABLE insert_into1 SELECT * from src ORDER BY key DESC LIMIT 10; + +select * from insert_into1 order by key; + +create table ctas_table as SELECT key, count(value) as foo from src GROUP BY key LIMIT 10; + +describe extended ctas_table; + +select * from ctas_table order by key; + + +set hive.exec.dynamic.partition=true; +SET hive.exec.dynamic.partition.mode=nonstrict; + +create table ctas_part (key int, value string) partitioned by (modkey bigint); + +insert overwrite table ctas_part partition (modkey) +select key, value, ceil(key / 100) from src where key is not null limit 10; + +select * from ctas_part order by key; + + + +DROP TABLE insert_into1; +DROP TABLE ctas_table; +DROP TABLE ctas_part; \ No newline at end of file diff --git a/ql/src/test/queries/clientpositive/join_vc.q b/ql/src/test/queries/clientpositive/join_vc.q index 63b3da7..8d7dea9 100644 --- a/ql/src/test/queries/clientpositive/join_vc.q +++ b/ql/src/test/queries/clientpositive/join_vc.q @@ -3,3 +3,10 @@ explain select t3.BLOCK__OFFSET__INSIDE__FILE,t3.key,t3.value from src t1 join src t2 on t1.key = t2.key join src t3 on t2.value = t3.value order by t3.BLOCK__OFFSET__INSIDE__FILE,t3.key,t3.value limit 3; select t3.BLOCK__OFFSET__INSIDE__FILE,t3.key,t3.value from src t1 join src t2 on t1.key = t2.key join src t3 on t2.value = t3.value order by t3.BLOCK__OFFSET__INSIDE__FILE,t3.key,t3.value limit 3; + +explain +select t2.BLOCK__OFFSET__INSIDE__FILE +from src t1 join src t2 on t1.key = t2.key where t1.key < 100; + +select t2.BLOCK__OFFSET__INSIDE__FILE +from src t1 join src t2 on t1.key = t2.key where t1.key < 100; diff --git a/ql/src/test/queries/clientpositive/optimize_nullscan.q b/ql/src/test/queries/clientpositive/optimize_nullscan.q index 61a71a2..f3b896b 100644 --- a/ql/src/test/queries/clientpositive/optimize_nullscan.q +++ b/ql/src/test/queries/clientpositive/optimize_nullscan.q @@ -23,3 +23,7 @@ select * from (select key from src where false) a left outer join (select value explain extended select * from (select key from src union all select src.key from src left outer join srcpart on src.key = srcpart.key) a where false; select * from (select key from src union all select src.key from src left outer join srcpart on src.key = srcpart.key) a where false; + +explain extended +select * from src s1, src s2 where false and s1.value = s2.value; +select * from src s1, src s2 where false and s1.value = s2.value; diff --git a/ql/src/test/queries/clientpositive/orc_vectorization_ppd.q b/ql/src/test/queries/clientpositive/orc_vectorization_ppd.q index 9bdad86..47112ad 100644 --- a/ql/src/test/queries/clientpositive/orc_vectorization_ppd.q +++ b/ql/src/test/queries/clientpositive/orc_vectorization_ppd.q @@ -13,12 +13,14 @@ stored as ORC tblproperties("orc.row.index.stride"="1000", "orc.stripe.size"="10 -- insert creates separate orc files insert overwrite table vectororc select "apple", "a", rand(1), "zoo" from srcorc; +set hive.optimize.constant.propagation=false; insert into table vectororc select null, "b", rand(2), "zoo" from srcorc; insert into table vectororc select null, "c", rand(3), "zoo" from srcorc; insert into table vectororc select "apple", "d", rand(4), "zoo" from srcorc; insert into table vectororc select null, "e", rand(5), "z" from srcorc; insert into table vectororc select "apple", "f", rand(6), "z" from srcorc; insert into table vectororc select null, "g", rand(7), "zoo" from srcorc; +set hive.optimize.constant.propagation=true; -- since vectororc table has multiple orc file we will load them into a single file using another table create table if not exists testorc diff --git a/ql/src/test/queries/clientpositive/partition_boolexpr.q b/ql/src/test/queries/clientpositive/partition_boolexpr.q new file mode 100644 index 0000000..3394d7d --- /dev/null +++ b/ql/src/test/queries/clientpositive/partition_boolexpr.q @@ -0,0 +1,12 @@ +-- create testing table. +create table part_boolexpr(key int, value string) partitioned by (dt int, ts string); + +-- both the below queries should return 0 rows +select count(*) from part_boolexpr where key = 'abc'; +select * from part_boolexpr where dt = 'abc'; +explain select count(1) from srcpart where true; +explain select count(1) from srcpart where false; +explain select count(1) from srcpart where true and hr='11'; +explain select count(1) from srcpart where true or hr='11'; +explain select count(1) from srcpart where false or hr='11'; +explain select count(1) from srcpart where false and hr='11'; \ No newline at end of file diff --git a/ql/src/test/queries/clientpositive/pcr.q b/ql/src/test/queries/clientpositive/pcr.q index 3be0ff2..a2ffda3 100644 --- a/ql/src/test/queries/clientpositive/pcr.q +++ b/ql/src/test/queries/clientpositive/pcr.q @@ -138,4 +138,6 @@ insert overwrite table foo_field partition (ds=7) select strct from ab where str select s,ds from foo_field where ((ds + s.a) > 0) order by ds,s; drop table foo_field; - +explain select key,value from srcpart where cast(hr as double) = cast(11 as double); +explain select key,value from srcpart where hr = cast(11 as double); +explain select key,value from srcpart where cast(hr as double) = 11 ; diff --git a/ql/src/test/queries/clientpositive/ptf_matchpath.q b/ql/src/test/queries/clientpositive/ptf_matchpath.q index 0cde350..2c1d766 100644 --- a/ql/src/test/queries/clientpositive/ptf_matchpath.q +++ b/ql/src/test/queries/clientpositive/ptf_matchpath.q @@ -32,5 +32,15 @@ from matchpath(on arg2('LATE'), arg3(arr_delay > 15), arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') ) -where fl_num = 1142; +where fl_num = 1142; + +-- 3. empty partition. +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + (select * from flights_tiny where fl_num = -1142) flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ); \ No newline at end of file diff --git a/ql/src/test/queries/clientpositive/select_same_col.q b/ql/src/test/queries/clientpositive/select_same_col.q new file mode 100644 index 0000000..d6902c2 --- /dev/null +++ b/ql/src/test/queries/clientpositive/select_same_col.q @@ -0,0 +1,19 @@ + +set hive.cbo.enable=true; + +drop table srclimit; +create table srclimit as select * from src limit 10; + +select cast(value as binary), value from srclimit; + +select cast(value as binary), value from srclimit order by value; + +select cast(value as binary), value from srclimit order by value limit 5; + +select cast(value as binary), value, key from srclimit order by value limit 5; + +select *, key, value from srclimit; + +select * from (select *, key, value from srclimit) t; + +drop table srclimit; \ No newline at end of file diff --git a/ql/src/test/queries/clientpositive/show_functions.q b/ql/src/test/queries/clientpositive/show_functions.q index c09f50a..106e05f 100644 --- a/ql/src/test/queries/clientpositive/show_functions.q +++ b/ql/src/test/queries/clientpositive/show_functions.q @@ -9,3 +9,23 @@ SHOW FUNCTIONS 'log.*'; SHOW FUNCTIONS '.*date.*'; SHOW FUNCTIONS '***'; + +SHOW FUNCTIONS LIKE 'When'; + +SHOW FUNCTIONS LIKE 'max|min'; + +SHOW FUNCTIONS LIKE 'xpath*|m*'; + +SHOW FUNCTIONS LIKE 'nomatch'; + +SHOW FUNCTIONS LIKE "log"; + +SHOW FUNCTIONS LIKE 'log'; + +SHOW FUNCTIONS LIKE `log`; + +SHOW FUNCTIONS LIKE 'log*'; + +SHOW FUNCTIONS LIKE "log*"; + +SHOW FUNCTIONS LIKE `log*`; diff --git a/ql/src/test/queries/clientpositive/sum_expr_with_order.q b/ql/src/test/queries/clientpositive/sum_expr_with_order.q new file mode 100644 index 0000000..4de9837 --- /dev/null +++ b/ql/src/test/queries/clientpositive/sum_expr_with_order.q @@ -0,0 +1,5 @@ + +select +cast(sum(key)*100 as decimal(15,3)) as c1 +from src +order by c1; diff --git a/ql/src/test/queries/clientpositive/tez_smb_1.q b/ql/src/test/queries/clientpositive/tez_smb_1.q index b675eea..763f7c3 100644 --- a/ql/src/test/queries/clientpositive/tez_smb_1.q +++ b/ql/src/test/queries/clientpositive/tez_smb_1.q @@ -33,6 +33,3 @@ set hive.auto.convert.join.noconditionaltask.size=500; explain select count(*) from tab s1 join tab s3 on s1.key=s3.key; -select s1.key, s1.value, s3.value from tab s1 join tab s3 on s1.key=s3.key; -select count(*) from tab s2; - diff --git a/ql/src/test/queries/clientpositive/timestamp_comparison2.q b/ql/src/test/queries/clientpositive/timestamp_comparison2.q new file mode 100644 index 0000000..d41cc83 --- /dev/null +++ b/ql/src/test/queries/clientpositive/timestamp_comparison2.q @@ -0,0 +1,23 @@ +-- Test timestamp-to-numeric comparison +select count(*) +FROM alltypesorc +WHERE +((ctinyint != 0) + AND + (((ctimestamp1 <= 0) + OR ((ctinyint = cint) OR (cstring2 LIKE 'ss'))) + AND ((988888 < cdouble) + OR ((ctimestamp2 > -29071) AND (3569 >= cdouble))))) +; + +-- Should have same result as previous query +select count(*) +FROM alltypesorc +WHERE +((ctinyint != 0) + AND + (((ctimestamp1 <= timestamp('1969-12-31 16:00:00')) + OR ((ctinyint = cint) OR (cstring2 LIKE 'ss'))) + AND ((988888 < cdouble) + OR ((ctimestamp2 > timestamp('1969-12-31 07:55:29')) AND (3569 >= cdouble))))) +; diff --git a/ql/src/test/queries/clientpositive/transform_acid.q b/ql/src/test/queries/clientpositive/transform_acid.q new file mode 100644 index 0000000..4cb9e38 --- /dev/null +++ b/ql/src/test/queries/clientpositive/transform_acid.q @@ -0,0 +1,13 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.enforce.bucketing=true; + +-- EXCLUDE_OS_WINDOWS + +create table transform_acid(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true'); +insert into table transform_acid select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint < 0 order by cint limit 1; + + +ADD FILE ../../ql/src/test/scripts/transform_acid_grep.sh; + +SELECT transform(*) USING 'transform_acid_grep.sh' AS (col string) FROM transform_acid; diff --git a/ql/src/test/queries/clientpositive/union_date_trim.q b/ql/src/test/queries/clientpositive/union_date_trim.q new file mode 100644 index 0000000..6842e56 --- /dev/null +++ b/ql/src/test/queries/clientpositive/union_date_trim.q @@ -0,0 +1,7 @@ +drop table if exists testDate; +create table testDate(id int, dt date); +insert into table testDate select 1, '2014-04-07' from src where key=100 limit 1; +insert into table testDate select 2, '2014-04-08' from src where key=100 limit 1; +insert into table testDate select 3, '2014-04-09' from src where key=100 limit 1; +--- without the fix following query will throw HiveException: Incompatible types for union operator +insert into table testDate select id, tm from (select id, dt as tm from testDate where id = 1 union all select id, dt as tm from testDate where id = 2 union all select id, trim(Cast (dt as string)) as tm from testDate where id = 3 ) a; diff --git a/ql/src/test/queries/clientpositive/vector_bucket.q b/ql/src/test/queries/clientpositive/vector_bucket.q new file mode 100644 index 0000000..19a09c4 --- /dev/null +++ b/ql/src/test/queries/clientpositive/vector_bucket.q @@ -0,0 +1,11 @@ +SET hive.vectorized.execution.enabled=true; +set hive.support.concurrency=true; +set hive.enforce.bucketing=true; + +CREATE TABLE non_orc_table(a INT, b STRING) CLUSTERED BY(a) INTO 2 BUCKETS STORED AS sequencefile; + + +explain +insert into table non_orc_table values(1, 'one'),(1, 'one'), (2, 'two'),(3, 'three'); select a, b from non_orc_table order by a; + +insert into table non_orc_table values(1, 'one'),(1, 'one'), (2, 'two'),(3, 'three'); select a, b from non_orc_table order by a; diff --git a/ql/src/test/queries/clientpositive/vector_data_types.q b/ql/src/test/queries/clientpositive/vector_data_types.q index 4c38968..61356fa 100644 --- a/ql/src/test/queries/clientpositive/vector_data_types.q +++ b/ql/src/test/queries/clientpositive/vector_data_types.q @@ -40,8 +40,14 @@ EXPLAIN SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i LIMIT 20; +SELECT SUM(HASH(*)) +FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i) as q; + SET hive.vectorized.execution.enabled=true; EXPLAIN select t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i LIMIT 20; -SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i LIMIT 20; \ No newline at end of file +SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i LIMIT 20; + +SELECT SUM(HASH(*)) +FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i) as q; \ No newline at end of file diff --git a/ql/src/test/queries/clientpositive/vector_groupby_reduce.q b/ql/src/test/queries/clientpositive/vector_groupby_reduce.q new file mode 100644 index 0000000..5da2089 --- /dev/null +++ b/ql/src/test/queries/clientpositive/vector_groupby_reduce.q @@ -0,0 +1,129 @@ +SET hive.vectorized.execution.enabled=true; + +create table store_sales_txt +( + ss_sold_date_sk int, + ss_sold_time_sk int, + ss_item_sk int, + ss_customer_sk int, + ss_cdemo_sk int, + ss_hdemo_sk int, + ss_addr_sk int, + ss_store_sk int, + ss_promo_sk int, + ss_ticket_number int, + ss_quantity int, + ss_wholesale_cost float, + ss_list_price float, + ss_sales_price float, + ss_ext_discount_amt float, + ss_ext_sales_price float, + ss_ext_wholesale_cost float, + ss_ext_list_price float, + ss_ext_tax float, + ss_coupon_amt float, + ss_net_paid float, + ss_net_paid_inc_tax float, + ss_net_profit float +) +row format delimited fields terminated by '|' +stored as textfile; + +LOAD DATA LOCAL INPATH '../../data/files/store_sales.txt' OVERWRITE INTO TABLE store_sales_txt; + +create table store_sales +( + ss_sold_date_sk int, + ss_sold_time_sk int, + ss_item_sk int, + ss_customer_sk int, + ss_cdemo_sk int, + ss_hdemo_sk int, + ss_addr_sk int, + ss_store_sk int, + ss_promo_sk int, + ss_ticket_number int, + ss_quantity int, + ss_wholesale_cost float, + ss_list_price float, + ss_sales_price float, + ss_ext_discount_amt float, + ss_ext_sales_price float, + ss_ext_wholesale_cost float, + ss_ext_list_price float, + ss_ext_tax float, + ss_coupon_amt float, + ss_net_paid float, + ss_net_paid_inc_tax float, + ss_net_profit float +) +stored as orc +tblproperties ("orc.stripe.size"="33554432", "orc.compress.size"="16384"); + +set hive.exec.dynamic.partition.mode=nonstrict; + +insert overwrite table store_sales +select +ss_sold_date_sk , + ss_sold_time_sk , + ss_item_sk , + ss_customer_sk , + ss_cdemo_sk , + ss_hdemo_sk , + ss_addr_sk , + ss_store_sk , + ss_promo_sk , + ss_ticket_number , + ss_quantity , + ss_wholesale_cost , + ss_list_price , + ss_sales_price , + ss_ext_discount_amt , + ss_ext_sales_price , + ss_ext_wholesale_cost , + ss_ext_list_price , + ss_ext_tax , + ss_coupon_amt , + ss_net_paid , + ss_net_paid_inc_tax , + ss_net_profit + from store_sales_txt; + +explain +select + ss_ticket_number +from + store_sales +group by ss_ticket_number +limit 20; + +select + ss_ticket_number +from + store_sales +group by ss_ticket_number +limit 20; + +explain +select + min(ss_ticket_number) +from + (select + ss_ticket_number + from + store_sales + group by ss_ticket_number) a +group by ss_ticket_number +limit 20; + +select + min(ss_ticket_number) +from + (select + ss_ticket_number + from + store_sales + group by ss_ticket_number) a +group by ss_ticket_number +limit 20; + diff --git a/ql/src/test/queries/clientpositive/vector_partitioned_date_time.q b/ql/src/test/queries/clientpositive/vector_partitioned_date_time.q new file mode 100644 index 0000000..1aeec8c --- /dev/null +++ b/ql/src/test/queries/clientpositive/vector_partitioned_date_time.q @@ -0,0 +1,127 @@ +set hive.fetch.task.conversion=minimal; + + +-- Check if vectorization code is handling partitioning on DATE and the other data types. + + +CREATE TABLE flights_tiny ( + origin_city_name STRING, + dest_city_name STRING, + fl_date DATE, + arr_delay FLOAT, + fl_num INT +); + +LOAD DATA LOCAL INPATH '../../data/files/flights_tiny.txt.1' OVERWRITE INTO TABLE flights_tiny; + +CREATE TABLE flights_tiny_orc STORED AS ORC AS +SELECT origin_city_name, dest_city_name, fl_date, to_utc_timestamp(fl_date, 'America/Los_Angeles') as fl_time, arr_delay, fl_num +FROM flights_tiny; + +SELECT * FROM flights_tiny_orc; + +SET hive.vectorized.execution.enabled=false; + +select * from flights_tiny_orc sort by fl_num, fl_date limit 25; + +select fl_date, count(*) from flights_tiny_orc group by fl_date; + +SET hive.vectorized.execution.enabled=true; + +explain +select * from flights_tiny_orc sort by fl_num, fl_date limit 25; + +select * from flights_tiny_orc sort by fl_num, fl_date limit 25; + +explain +select fl_date, count(*) from flights_tiny_orc group by fl_date; + +select fl_date, count(*) from flights_tiny_orc group by fl_date; + + +SET hive.vectorized.execution.enabled=false; + +CREATE TABLE flights_tiny_orc_partitioned_date ( + origin_city_name STRING, + dest_city_name STRING, + fl_time TIMESTAMP, + arr_delay FLOAT, + fl_num INT +) +PARTITIONED BY (fl_date DATE) +STORED AS ORC; + +set hive.exec.dynamic.partition.mode=nonstrict; + +INSERT INTO TABLE flights_tiny_orc_partitioned_date +PARTITION (fl_date) +SELECT origin_city_name, dest_city_name, fl_time, arr_delay, fl_num, fl_date +FROM flights_tiny_orc; + + +select * from flights_tiny_orc_partitioned_date; + +select * from flights_tiny_orc_partitioned_date sort by fl_num, fl_date limit 25; + +select fl_date, count(*) from flights_tiny_orc_partitioned_date group by fl_date; + +SET hive.vectorized.execution.enabled=true; + +explain +select * from flights_tiny_orc_partitioned_date; + +select * from flights_tiny_orc_partitioned_date; + +explain +select * from flights_tiny_orc_partitioned_date sort by fl_num, fl_date limit 25; + +select * from flights_tiny_orc_partitioned_date sort by fl_num, fl_date limit 25; + +explain +select fl_date, count(*) from flights_tiny_orc_partitioned_date group by fl_date; + +select fl_date, count(*) from flights_tiny_orc_partitioned_date group by fl_date; + + +SET hive.vectorized.execution.enabled=false; + +CREATE TABLE flights_tiny_orc_partitioned_timestamp ( + origin_city_name STRING, + dest_city_name STRING, + fl_date DATE, + arr_delay FLOAT, + fl_num INT +) +PARTITIONED BY (fl_time TIMESTAMP) +STORED AS ORC; + +set hive.exec.dynamic.partition.mode=nonstrict; + +INSERT INTO TABLE flights_tiny_orc_partitioned_timestamp +PARTITION (fl_time) +SELECT origin_city_name, dest_city_name, fl_date, arr_delay, fl_num, fl_time +FROM flights_tiny_orc; + + +select * from flights_tiny_orc_partitioned_timestamp; + +select * from flights_tiny_orc_partitioned_timestamp sort by fl_num, fl_time limit 25; + +select fl_time, count(*) from flights_tiny_orc_partitioned_timestamp group by fl_time; + +SET hive.vectorized.execution.enabled=true; + +explain +select * from flights_tiny_orc_partitioned_timestamp; + +select * from flights_tiny_orc_partitioned_timestamp; + +explain +select * from flights_tiny_orc_partitioned_timestamp sort by fl_num, fl_time limit 25; + +select * from flights_tiny_orc_partitioned_timestamp sort by fl_num, fl_time limit 25; + +explain +select fl_time, count(*) from flights_tiny_orc_partitioned_timestamp group by fl_time; + +select fl_time, count(*) from flights_tiny_orc_partitioned_timestamp group by fl_time; \ No newline at end of file diff --git a/ql/src/test/queries/clientpositive/vector_reduce_groupby_decimal.q b/ql/src/test/queries/clientpositive/vector_reduce_groupby_decimal.q new file mode 100644 index 0000000..a282518 --- /dev/null +++ b/ql/src/test/queries/clientpositive/vector_reduce_groupby_decimal.q @@ -0,0 +1,35 @@ +CREATE TABLE decimal_test STORED AS ORC AS SELECT cint, cdouble, CAST (((cdouble*22.1)/37) AS DECIMAL(20,10)) AS cdecimal1, CAST (((cdouble*9.3)/13) AS DECIMAL(23,14)) AS cdecimal2 FROM alltypesorc +WHERE cint is not null and cdouble is not null; + +SET hive.vectorized.execution.enabled=true; + +EXPLAIN +SELECT cint, cdouble, cdecimal1, cdecimal2, min(cdecimal1) as min_decimal1 FROM decimal_test +WHERE cdecimal1 is not null and cdecimal2 is not null +GROUP BY cint, cdouble, cdecimal1, cdecimal2 +ORDER BY cint, cdouble, cdecimal1, cdecimal2 +LIMIT 50; + +SELECT cint, cdouble, cdecimal1, cdecimal2, min(cdecimal1) as min_decimal1 FROM decimal_test +WHERE cdecimal1 is not null and cdecimal2 is not null +GROUP BY cint, cdouble, cdecimal1, cdecimal2 +ORDER BY cint, cdouble, cdecimal1, cdecimal2 +LIMIT 50; + +SET hive.vectorized.execution.enabled=false; + +SELECT sum(hash(*)) + FROM (SELECT cint, cdouble, cdecimal1, cdecimal2, min(cdecimal1) as min_decimal1 FROM decimal_test + WHERE cdecimal1 is not null and cdecimal2 is not null + GROUP BY cint, cdouble, cdecimal1, cdecimal2 + ORDER BY cint, cdouble, cdecimal1, cdecimal2 + LIMIT 50) as q; + +SET hive.vectorized.execution.enabled=true; + +SELECT sum(hash(*)) + FROM (SELECT cint, cdouble, cdecimal1, cdecimal2, min(cdecimal1) as min_decimal1 FROM decimal_test + WHERE cdecimal1 is not null and cdecimal2 is not null + GROUP BY cint, cdouble, cdecimal1, cdecimal2 + ORDER BY cint, cdouble, cdecimal1, cdecimal2 + LIMIT 50) as q; diff --git a/ql/src/test/queries/clientpositive/windowing.q b/ql/src/test/queries/clientpositive/windowing.q index 3f5c3bf..8dceafa 100644 --- a/ql/src/test/queries/clientpositive/windowing.q +++ b/ql/src/test/queries/clientpositive/windowing.q @@ -444,3 +444,7 @@ select p_retailprice, avg(p_retailprice) over (partition by p_mfgr order by p_na sum(p_retailprice) over (partition by p_mfgr order by p_name rows between current row and 6 following) from part where p_mfgr='Manufacturer#1'; + +-- 47. empty partition +select sum(p_size) over (partition by p_mfgr ) +from part where p_mfgr = 'm1'; diff --git a/ql/src/test/results/beelinepositive/avro_schema_error_message.q.out b/ql/src/test/results/beelinepositive/avro_schema_error_message.q.out index 0dfc75a..e69de29 100644 --- a/ql/src/test/results/beelinepositive/avro_schema_error_message.q.out +++ b/ql/src/test/results/beelinepositive/avro_schema_error_message.q.out @@ -1,24 +0,0 @@ -Saving all output to "!!{outputDirectory}!!/avro_schema_error_message.q.raw". Enter "record" with no arguments to stop it. ->>> !run !!{qFileDirectory}!!/avro_schema_error_message.q ->>> -- verify we get the sentinel schema if we don't provide one ->>> ->>> CREATE TABLE avro_with_no_schema -ROW FORMAT -SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' -STORED AS -INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' -OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat'; -No rows affected ->>> ->>> DESCRIBE avro_with_no_schema; -'col_name','data_type','comment' -'error_error_error_error_error_error_error','string','from deserializer' -'cannot_determine_schema','string','from deserializer' -'check','string','from deserializer' -'schema','string','from deserializer' -'url','string','from deserializer' -'and','string','from deserializer' -'literal','string','from deserializer' -7 rows selected ->>> ->>> !record diff --git a/ql/src/test/results/clientnegative/alter_concatenate_indexed_table.q.out b/ql/src/test/results/clientnegative/alter_concatenate_indexed_table.q.out index 19fbf51..7461ba3 100644 --- a/ql/src/test/results/clientnegative/alter_concatenate_indexed_table.q.out +++ b/ql/src/test/results/clientnegative/alter_concatenate_indexed_table.q.out @@ -76,5 +76,5 @@ PREHOOK: query: show indexes on src_rc_concatenate_test PREHOOK: type: SHOWINDEXES POSTHOOK: query: show indexes on src_rc_concatenate_test POSTHOOK: type: SHOWINDEXES -src_rc_concatenate_test_index src_rc_concatenate_test key default__src_rc_concatenate_test_src_rc_concatenate_test_index__ compact +src_rc_concatenate_test_index src_rc_concatenate_test key default.default__src_rc_concatenate_test_src_rc_concatenate_test_index__ compact FAILED: SemanticException org.apache.hadoop.hive.ql.parse.SemanticException: can not do merge because source table default.src_rc_concatenate_test is indexed. diff --git a/ql/src/test/results/clientnegative/alter_partition_partial_spec_dyndisabled.q.out b/ql/src/test/results/clientnegative/alter_partition_partial_spec_dyndisabled.q.out new file mode 100644 index 0000000..8ec2c28 --- /dev/null +++ b/ql/src/test/results/clientnegative/alter_partition_partial_spec_dyndisabled.q.out @@ -0,0 +1,40 @@ +PREHOOK: query: create table alter_partition_partial_spec_dyndisabled0 (c1 string) partitioned by (p1 string, p2 string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@alter_partition_partial_spec_dyndisabled0 +POSTHOOK: query: create table alter_partition_partial_spec_dyndisabled0 (c1 string) partitioned by (p1 string, p2 string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@alter_partition_partial_spec_dyndisabled0 +PREHOOK: query: alter table alter_partition_partial_spec_dyndisabled0 add partition (p1='abc', p2='123') +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@alter_partition_partial_spec_dyndisabled0 +POSTHOOK: query: alter table alter_partition_partial_spec_dyndisabled0 add partition (p1='abc', p2='123') +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@alter_partition_partial_spec_dyndisabled0 +POSTHOOK: Output: default@alter_partition_partial_spec_dyndisabled0@p1=abc/p2=123 +PREHOOK: query: alter table alter_partition_partial_spec_dyndisabled0 partition (p1, p2) change c1 c1 int +PREHOOK: type: ALTERTABLE_RENAMECOL +PREHOOK: Input: default@alter_partition_partial_spec_dyndisabled0 +PREHOOK: Output: default@alter_partition_partial_spec_dyndisabled0@p1=abc/p2=123 +POSTHOOK: query: alter table alter_partition_partial_spec_dyndisabled0 partition (p1, p2) change c1 c1 int +POSTHOOK: type: ALTERTABLE_RENAMECOL +POSTHOOK: Input: default@alter_partition_partial_spec_dyndisabled0 +POSTHOOK: Input: default@alter_partition_partial_spec_dyndisabled0@p1=abc/p2=123 +POSTHOOK: Output: default@alter_partition_partial_spec_dyndisabled0@p1=abc/p2=123 +PREHOOK: query: describe alter_partition_partial_spec_dyndisabled0 partition (p1='abc', p2='123') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@alter_partition_partial_spec_dyndisabled0 +POSTHOOK: query: describe alter_partition_partial_spec_dyndisabled0 partition (p1='abc', p2='123') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@alter_partition_partial_spec_dyndisabled0 +c1 int +p1 string +p2 string + +# Partition Information +# col_name data_type comment + +p1 string +p2 string +FAILED: SemanticException [Error 10095]: Dynamic partition is disabled. Either enable it by setting hive.exec.dynamic.partition=true or specify partition column values diff --git a/ql/src/test/results/clientnegative/ambiguous_col.q.out b/ql/src/test/results/clientnegative/ambiguous_col.q.out index 237c21f..a2915a4 100644 --- a/ql/src/test/results/clientnegative/ambiguous_col.q.out +++ b/ql/src/test/results/clientnegative/ambiguous_col.q.out @@ -1 +1 @@ -FAILED: SemanticException [Error 10007]: Ambiguous column reference key +FAILED: SemanticException [Error 10007]: Ambiguous column reference key in a diff --git a/ql/src/test/results/clientnegative/ambiguous_col0.q.out b/ql/src/test/results/clientnegative/ambiguous_col0.q.out index 237c21f..e69de29 100644 --- a/ql/src/test/results/clientnegative/ambiguous_col0.q.out +++ b/ql/src/test/results/clientnegative/ambiguous_col0.q.out @@ -1 +0,0 @@ -FAILED: SemanticException [Error 10007]: Ambiguous column reference key diff --git a/ql/src/test/results/clientnegative/ambiguous_col1.q.out b/ql/src/test/results/clientnegative/ambiguous_col1.q.out index 237c21f..e69de29 100644 --- a/ql/src/test/results/clientnegative/ambiguous_col1.q.out +++ b/ql/src/test/results/clientnegative/ambiguous_col1.q.out @@ -1 +0,0 @@ -FAILED: SemanticException [Error 10007]: Ambiguous column reference key diff --git a/ql/src/test/results/clientnegative/ambiguous_col2.q.out b/ql/src/test/results/clientnegative/ambiguous_col2.q.out index 237c21f..e69de29 100644 --- a/ql/src/test/results/clientnegative/ambiguous_col2.q.out +++ b/ql/src/test/results/clientnegative/ambiguous_col2.q.out @@ -1 +0,0 @@ -FAILED: SemanticException [Error 10007]: Ambiguous column reference key diff --git a/ql/src/test/results/clientnegative/authorization_disallow_transform.q.out b/ql/src/test/results/clientnegative/authorization_disallow_transform.q.out index 39819b6..812c1a2 100644 --- a/ql/src/test/results/clientnegative/authorization_disallow_transform.q.out +++ b/ql/src/test/results/clientnegative/authorization_disallow_transform.q.out @@ -2,9 +2,17 @@ PREHOOK: query: set role ALL PREHOOK: type: SHOW_ROLES POSTHOOK: query: set role ALL POSTHOOK: type: SHOW_ROLES -PREHOOK: query: SELECT TRANSFORM (*) USING 'cat' AS (key, value) FROM src +PREHOOK: query: create table t1(i int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t1 +POSTHOOK: query: create table t1(i int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t1 +PREHOOK: query: SELECT TRANSFORM (*) USING 'cat' AS (key, value) FROM t1 PREHOOK: type: QUERY -PREHOOK: Input: default@src +PREHOOK: Input: default@t1 #### A masked pattern was here #### FAILED: Hive Internal Error: org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAccessControlException(Query with transform clause is disallowed in current configuration.) org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAccessControlException: Query with transform clause is disallowed in current configuration. diff --git a/ql/src/test/results/clientnegative/authorization_set_invalidconf.q.out b/ql/src/test/results/clientnegative/authorization_set_invalidconf.q.out new file mode 100644 index 0000000..2a58c72 --- /dev/null +++ b/ql/src/test/results/clientnegative/authorization_set_invalidconf.q.out @@ -0,0 +1,9 @@ +PREHOOK: query: -- run a sql query to initialize authorization, then try setting a allowed config and then a disallowed config param +use default +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:default +POSTHOOK: query: -- run a sql query to initialize authorization, then try setting a allowed config and then a disallowed config param +use default +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:default +Query returned non-zero code: 1, cause: Cannot modify hive.security.authorization.enabled at runtime. It is not in list of params that are allowed to be modified at runtime diff --git a/ql/src/test/results/clientnegative/authorization_show_role_principals_no_admin.q.out b/ql/src/test/results/clientnegative/authorization_show_role_principals_no_admin.q.out index b0c7b75..659edcc 100644 --- a/ql/src/test/results/clientnegative/authorization_show_role_principals_no_admin.q.out +++ b/ql/src/test/results/clientnegative/authorization_show_role_principals_no_admin.q.out @@ -1,4 +1,4 @@ PREHOOK: query: -- This test will fail because hive_test_user is not in admin role show principals role1 PREHOOK: type: SHOW_ROLE_PRINCIPALS -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Current user : hive_test_user is not allowed get principals in a role. User has to belong to ADMIN role and have it as current role, for this action. +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Current user : hive_test_user is not allowed get principals in a role. User has to belong to ADMIN role and have it as current role, for this action. Otherwise, grantor need to have ADMIN OPTION on role being granted and have it as a current role for this action. diff --git a/ql/src/test/results/clientnegative/set_hiveconf_validation2.q.out b/ql/src/test/results/clientnegative/set_hiveconf_validation2.q.out index 33f9360..af004a1 100644 --- a/ql/src/test/results/clientnegative/set_hiveconf_validation2.q.out +++ b/ql/src/test/results/clientnegative/set_hiveconf_validation2.q.out @@ -1,11 +1,11 @@ -PREHOOK: query: -- should fail: hive.fetch.task.conversion accepts minimal or more +PREHOOK: query: -- should fail: hive.fetch.task.conversion accepts none, minimal or more desc src PREHOOK: type: DESCTABLE PREHOOK: Input: default@src -POSTHOOK: query: -- should fail: hive.fetch.task.conversion accepts minimal or more +POSTHOOK: query: -- should fail: hive.fetch.task.conversion accepts none, minimal or more desc src POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@src key string default value string default -Query returned non-zero code: 1, cause: 'SET hive.fetch.task.conversion=true' FAILED in validation : Invalid value.. expects one of [minimal, more]. +Query returned non-zero code: 1, cause: 'SET hive.fetch.task.conversion=true' FAILED in validation : Invalid value.. expects one of [none, minimal, more]. diff --git a/ql/src/test/results/clientpositive/acid_vectorization.q.out b/ql/src/test/results/clientpositive/acid_vectorization.q.out index 18dada5..1792979 100644 --- a/ql/src/test/results/clientpositive/acid_vectorization.q.out +++ b/ql/src/test/results/clientpositive/acid_vectorization.q.out @@ -42,3 +42,21 @@ POSTHOOK: query: delete from acid_vectorized where b = 'foo' POSTHOOK: type: QUERY POSTHOOK: Input: default@acid_vectorized POSTHOOK: Output: default@acid_vectorized +PREHOOK: query: select a, b from acid_vectorized order by a, b +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_vectorized +#### A masked pattern was here #### +POSTHOOK: query: select a, b from acid_vectorized order by a, b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_vectorized +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa +-1073051226 A34p7oRr2WvUJNf +-1072910839 0iqrc5 +-1072081801 dPkN74F7 +-1072076362 2uLyD28144vklju213J1mr +-1071480828 aw724t8c5558x2xneC624 +-1071363017 Anj0oF +-1070883071 0ruyd6Y50JpdGRf6HqD +-1070551679 iUR3Q +-1069736047 k17Am8uPHWk02cEf1jet diff --git a/ql/src/test/results/clientpositive/acid_vectorization_partition.q.out b/ql/src/test/results/clientpositive/acid_vectorization_partition.q.out new file mode 100644 index 0000000..ee97cc9 --- /dev/null +++ b/ql/src/test/results/clientpositive/acid_vectorization_partition.q.out @@ -0,0 +1,60 @@ +PREHOOK: query: CREATE TABLE acid_vectorized_part(a INT, b STRING) partitioned by (ds string) CLUSTERED BY(a) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_vectorized_part +POSTHOOK: query: CREATE TABLE acid_vectorized_part(a INT, b STRING) partitioned by (ds string) CLUSTERED BY(a) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_vectorized_part +PREHOOK: query: insert into table acid_vectorized_part partition (ds = 'today') select cint, cstring1 from alltypesorc where cint is not null order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_vectorized_part@ds=today +POSTHOOK: query: insert into table acid_vectorized_part partition (ds = 'today') select cint, cstring1 from alltypesorc where cint is not null order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_vectorized_part@ds=today +POSTHOOK: Lineage: acid_vectorized_part PARTITION(ds=today).a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_vectorized_part PARTITION(ds=today).b SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: insert into table acid_vectorized_part partition (ds = 'tomorrow') select cint, cstring1 from alltypesorc where cint is not null order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_vectorized_part@ds=tomorrow +POSTHOOK: query: insert into table acid_vectorized_part partition (ds = 'tomorrow') select cint, cstring1 from alltypesorc where cint is not null order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_vectorized_part@ds=tomorrow +POSTHOOK: Lineage: acid_vectorized_part PARTITION(ds=tomorrow).a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_vectorized_part PARTITION(ds=tomorrow).b SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: select * from acid_vectorized_part order by a, b +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_vectorized_part +PREHOOK: Input: default@acid_vectorized_part@ds=today +PREHOOK: Input: default@acid_vectorized_part@ds=tomorrow +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_vectorized_part order by a, b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_vectorized_part +POSTHOOK: Input: default@acid_vectorized_part@ds=today +POSTHOOK: Input: default@acid_vectorized_part@ds=tomorrow +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa today +-1073279343 oj1YrV5Wa tomorrow +-1073051226 A34p7oRr2WvUJNf tomorrow +-1073051226 A34p7oRr2WvUJNf today +-1072910839 0iqrc5 tomorrow +-1072910839 0iqrc5 today +-1072081801 dPkN74F7 today +-1072081801 dPkN74F7 tomorrow +-1072076362 2uLyD28144vklju213J1mr today +-1072076362 2uLyD28144vklju213J1mr tomorrow +-1071480828 aw724t8c5558x2xneC624 tomorrow +-1071480828 aw724t8c5558x2xneC624 today +-1071363017 Anj0oF today +-1071363017 Anj0oF tomorrow +-1070883071 0ruyd6Y50JpdGRf6HqD tomorrow +-1070883071 0ruyd6Y50JpdGRf6HqD today +-1070551679 iUR3Q today +-1070551679 iUR3Q tomorrow +-1069736047 k17Am8uPHWk02cEf1jet tomorrow +-1069736047 k17Am8uPHWk02cEf1jet today diff --git a/ql/src/test/results/clientpositive/acid_vectorization_project.q.out b/ql/src/test/results/clientpositive/acid_vectorization_project.q.out new file mode 100644 index 0000000..1bdacb9 --- /dev/null +++ b/ql/src/test/results/clientpositive/acid_vectorization_project.q.out @@ -0,0 +1,73 @@ +PREHOOK: query: CREATE TABLE acid_vectorized(a INT, b STRING, c float) CLUSTERED BY(a) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_vectorized +POSTHOOK: query: CREATE TABLE acid_vectorized(a INT, b STRING, c float) CLUSTERED BY(a) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_vectorized +PREHOOK: query: insert into table acid_vectorized select cint, cstring1, cfloat from alltypesorc where cint is not null order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_vectorized +POSTHOOK: query: insert into table acid_vectorized select cint, cstring1, cfloat from alltypesorc where cint is not null order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_vectorized +POSTHOOK: Lineage: acid_vectorized.a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_vectorized.b SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: acid_vectorized.c SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] +PREHOOK: query: select a,b from acid_vectorized order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_vectorized +#### A masked pattern was here #### +POSTHOOK: query: select a,b from acid_vectorized order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_vectorized +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa +-1073051226 A34p7oRr2WvUJNf +-1072910839 0iqrc5 +-1072081801 dPkN74F7 +-1072076362 2uLyD28144vklju213J1mr +-1071480828 aw724t8c5558x2xneC624 +-1071363017 Anj0oF +-1070883071 0ruyd6Y50JpdGRf6HqD +-1070551679 iUR3Q +-1069736047 k17Am8uPHWk02cEf1jet +PREHOOK: query: select a,c from acid_vectorized order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_vectorized +#### A masked pattern was here #### +POSTHOOK: query: select a,c from acid_vectorized order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_vectorized +#### A masked pattern was here #### +-1073279343 11.0 +-1073051226 NULL +-1072910839 11.0 +-1072081801 NULL +-1072076362 NULL +-1071480828 -51.0 +-1071363017 8.0 +-1070883071 NULL +-1070551679 NULL +-1069736047 11.0 +PREHOOK: query: select b,c from acid_vectorized order by b +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_vectorized +#### A masked pattern was here #### +POSTHOOK: query: select b,c from acid_vectorized order by b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_vectorized +#### A masked pattern was here #### +0iqrc5 11.0 +0ruyd6Y50JpdGRf6HqD NULL +2uLyD28144vklju213J1mr NULL +A34p7oRr2WvUJNf NULL +Anj0oF 8.0 +aw724t8c5558x2xneC624 -51.0 +dPkN74F7 NULL +iUR3Q NULL +k17Am8uPHWk02cEf1jet 11.0 +oj1YrV5Wa 11.0 diff --git a/ql/src/test/results/clientpositive/alter_concatenate_indexed_table.q.out b/ql/src/test/results/clientpositive/alter_concatenate_indexed_table.q.out index ffcbcf9..7f8458d 100644 --- a/ql/src/test/results/clientpositive/alter_concatenate_indexed_table.q.out +++ b/ql/src/test/results/clientpositive/alter_concatenate_indexed_table.q.out @@ -76,7 +76,7 @@ PREHOOK: query: show indexes on src_rc_concatenate_test PREHOOK: type: SHOWINDEXES POSTHOOK: query: show indexes on src_rc_concatenate_test POSTHOOK: type: SHOWINDEXES -src_rc_concatenate_test_index src_rc_concatenate_test key default__src_rc_concatenate_test_src_rc_concatenate_test_index__ compact +src_rc_concatenate_test_index src_rc_concatenate_test key default.default__src_rc_concatenate_test_src_rc_concatenate_test_index__ compact PREHOOK: query: alter table src_rc_concatenate_test concatenate PREHOOK: type: ALTER_TABLE_MERGE PREHOOK: Input: default@src_rc_concatenate_test @@ -215,7 +215,7 @@ PREHOOK: query: show indexes on src_rc_concatenate_test_part PREHOOK: type: SHOWINDEXES POSTHOOK: query: show indexes on src_rc_concatenate_test_part POSTHOOK: type: SHOWINDEXES -src_rc_concatenate_test_part_index src_rc_concatenate_test_part key default__src_rc_concatenate_test_part_src_rc_concatenate_test_part_index__ compact +src_rc_concatenate_test_part_index src_rc_concatenate_test_part key default.default__src_rc_concatenate_test_part_src_rc_concatenate_test_part_index__ compact PREHOOK: query: alter table src_rc_concatenate_test_part partition (ds='2011') concatenate PREHOOK: type: ALTER_PARTITION_MERGE PREHOOK: Input: default@src_rc_concatenate_test_part diff --git a/ql/src/test/results/clientpositive/alter_partition_change_col.q.out b/ql/src/test/results/clientpositive/alter_partition_change_col.q.out index 7123e40..148bff4 100644 --- a/ql/src/test/results/clientpositive/alter_partition_change_col.q.out +++ b/ql/src/test/results/clientpositive/alter_partition_change_col.q.out @@ -18,73 +18,81 @@ POSTHOOK: query: load data local inpath '../../data/files/dec.txt' overwrite int POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@alter_partition_change_col0 -PREHOOK: query: create table alter_partition_change_col1 (c1 string, c2 string) partitioned by (p1 string) +PREHOOK: query: create table alter_partition_change_col1 (c1 string, c2 string) partitioned by (p1 string, p2 string) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@alter_partition_change_col1 -POSTHOOK: query: create table alter_partition_change_col1 (c1 string, c2 string) partitioned by (p1 string) +POSTHOOK: query: create table alter_partition_change_col1 (c1 string, c2 string) partitioned by (p1 string, p2 string) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@alter_partition_change_col1 -PREHOOK: query: insert overwrite table alter_partition_change_col1 partition (p1) - select c1, c2, 'abc' from alter_partition_change_col0 +PREHOOK: query: insert overwrite table alter_partition_change_col1 partition (p1, p2) + select c1, c2, 'abc', '123' from alter_partition_change_col0 union all - select c1, c2, null from alter_partition_change_col0 + select c1, c2, null, '123' from alter_partition_change_col0 PREHOOK: type: QUERY PREHOOK: Input: default@alter_partition_change_col0 PREHOOK: Output: default@alter_partition_change_col1 -POSTHOOK: query: insert overwrite table alter_partition_change_col1 partition (p1) - select c1, c2, 'abc' from alter_partition_change_col0 +POSTHOOK: query: insert overwrite table alter_partition_change_col1 partition (p1, p2) + select c1, c2, 'abc', '123' from alter_partition_change_col0 union all - select c1, c2, null from alter_partition_change_col0 + select c1, c2, null, '123' from alter_partition_change_col0 POSTHOOK: type: QUERY POSTHOOK: Input: default@alter_partition_change_col0 -POSTHOOK: Output: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__ -POSTHOOK: Output: default@alter_partition_change_col1@p1=abc -POSTHOOK: Lineage: alter_partition_change_col1 PARTITION(p1=__HIVE_DEFAULT_PARTITION__).c1 EXPRESSION [(alter_partition_change_col0)alter_partition_change_col0.FieldSchema(name:c1, type:string, comment:null), (alter_partition_change_col0)alter_partition_change_col0.FieldSchema(name:c1, type:string, comment:null), ] -POSTHOOK: Lineage: alter_partition_change_col1 PARTITION(p1=__HIVE_DEFAULT_PARTITION__).c2 EXPRESSION [(alter_partition_change_col0)alter_partition_change_col0.FieldSchema(name:c2, type:string, comment:null), (alter_partition_change_col0)alter_partition_change_col0.FieldSchema(name:c2, type:string, comment:null), ] -POSTHOOK: Lineage: alter_partition_change_col1 PARTITION(p1=abc).c1 EXPRESSION [(alter_partition_change_col0)alter_partition_change_col0.FieldSchema(name:c1, type:string, comment:null), (alter_partition_change_col0)alter_partition_change_col0.FieldSchema(name:c1, type:string, comment:null), ] -POSTHOOK: Lineage: alter_partition_change_col1 PARTITION(p1=abc).c2 EXPRESSION [(alter_partition_change_col0)alter_partition_change_col0.FieldSchema(name:c2, type:string, comment:null), (alter_partition_change_col0)alter_partition_change_col0.FieldSchema(name:c2, type:string, comment:null), ] +POSTHOOK: Output: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__/p2=123 +POSTHOOK: Output: default@alter_partition_change_col1@p1=abc/p2=123 +POSTHOOK: Lineage: alter_partition_change_col1 PARTITION(p1=__HIVE_DEFAULT_PARTITION__,p2=123).c1 EXPRESSION [(alter_partition_change_col0)alter_partition_change_col0.FieldSchema(name:c1, type:string, comment:null), (alter_partition_change_col0)alter_partition_change_col0.FieldSchema(name:c1, type:string, comment:null), ] +POSTHOOK: Lineage: alter_partition_change_col1 PARTITION(p1=__HIVE_DEFAULT_PARTITION__,p2=123).c2 EXPRESSION [(alter_partition_change_col0)alter_partition_change_col0.FieldSchema(name:c2, type:string, comment:null), (alter_partition_change_col0)alter_partition_change_col0.FieldSchema(name:c2, type:string, comment:null), ] +POSTHOOK: Lineage: alter_partition_change_col1 PARTITION(p1=abc,p2=123).c1 EXPRESSION [(alter_partition_change_col0)alter_partition_change_col0.FieldSchema(name:c1, type:string, comment:null), (alter_partition_change_col0)alter_partition_change_col0.FieldSchema(name:c1, type:string, comment:null), ] +POSTHOOK: Lineage: alter_partition_change_col1 PARTITION(p1=abc,p2=123).c2 EXPRESSION [(alter_partition_change_col0)alter_partition_change_col0.FieldSchema(name:c2, type:string, comment:null), (alter_partition_change_col0)alter_partition_change_col0.FieldSchema(name:c2, type:string, comment:null), ] PREHOOK: query: show partitions alter_partition_change_col1 PREHOOK: type: SHOWPARTITIONS PREHOOK: Input: default@alter_partition_change_col1 POSTHOOK: query: show partitions alter_partition_change_col1 POSTHOOK: type: SHOWPARTITIONS POSTHOOK: Input: default@alter_partition_change_col1 -p1=__HIVE_DEFAULT_PARTITION__ -p1=abc -PREHOOK: query: select * from alter_partition_change_col1 +p1=__HIVE_DEFAULT_PARTITION__/p2=123 +p1=abc/p2=123 +PREHOOK: query: select * from alter_partition_change_col1 where p1='abc' PREHOOK: type: QUERY PREHOOK: Input: default@alter_partition_change_col1 -PREHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__ -PREHOOK: Input: default@alter_partition_change_col1@p1=abc +PREHOOK: Input: default@alter_partition_change_col1@p1=abc/p2=123 #### A masked pattern was here #### -POSTHOOK: query: select * from alter_partition_change_col1 +POSTHOOK: query: select * from alter_partition_change_col1 where p1='abc' POSTHOOK: type: QUERY POSTHOOK: Input: default@alter_partition_change_col1 -POSTHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__ -POSTHOOK: Input: default@alter_partition_change_col1@p1=abc -#### A masked pattern was here #### -Beck 0.0 __HIVE_DEFAULT_PARTITION__ -Beck 0.0 abc -Beck 77.341 __HIVE_DEFAULT_PARTITION__ -Beck 77.341 abc -Beck 79.9 __HIVE_DEFAULT_PARTITION__ -Beck 79.9 abc -Cluck 5.96 __HIVE_DEFAULT_PARTITION__ -Cluck 5.96 abc -Mary 33.33 __HIVE_DEFAULT_PARTITION__ -Mary 33.33 abc -Mary 4.329 __HIVE_DEFAULT_PARTITION__ -Mary 4.329 abc -Snow 55.71 __HIVE_DEFAULT_PARTITION__ -Snow 55.71 abc -Tom -12.25 __HIVE_DEFAULT_PARTITION__ -Tom -12.25 abc -Tom 19.00 __HIVE_DEFAULT_PARTITION__ -Tom 19.00 abc -Tom 234.79 __HIVE_DEFAULT_PARTITION__ -Tom 234.79 abc +POSTHOOK: Input: default@alter_partition_change_col1@p1=abc/p2=123 +#### A masked pattern was here #### +Beck 0.0 abc 123 +Beck 77.341 abc 123 +Beck 79.9 abc 123 +Cluck 5.96 abc 123 +Mary 33.33 abc 123 +Mary 4.329 abc 123 +Snow 55.71 abc 123 +Tom -12.25 abc 123 +Tom 19.00 abc 123 +Tom 234.79 abc 123 +PREHOOK: query: select * from alter_partition_change_col1 where p1='__HIVE_DEFAULT_PARTITION__' +PREHOOK: type: QUERY +PREHOOK: Input: default@alter_partition_change_col1 +PREHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__/p2=123 +#### A masked pattern was here #### +POSTHOOK: query: select * from alter_partition_change_col1 where p1='__HIVE_DEFAULT_PARTITION__' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alter_partition_change_col1 +POSTHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__/p2=123 +#### A masked pattern was here #### +Beck 0.0 __HIVE_DEFAULT_PARTITION__ 123 +Beck 77.341 __HIVE_DEFAULT_PARTITION__ 123 +Beck 79.9 __HIVE_DEFAULT_PARTITION__ 123 +Cluck 5.96 __HIVE_DEFAULT_PARTITION__ 123 +Mary 33.33 __HIVE_DEFAULT_PARTITION__ 123 +Mary 4.329 __HIVE_DEFAULT_PARTITION__ 123 +Snow 55.71 __HIVE_DEFAULT_PARTITION__ 123 +Tom -12.25 __HIVE_DEFAULT_PARTITION__ 123 +Tom 19.00 __HIVE_DEFAULT_PARTITION__ 123 +Tom 234.79 __HIVE_DEFAULT_PARTITION__ 123 PREHOOK: query: -- Change c2 to decimal(10,0) alter table alter_partition_change_col1 change c2 c2 decimal(10,0) PREHOOK: type: ALTERTABLE_RENAMECOL @@ -95,56 +103,64 @@ alter table alter_partition_change_col1 change c2 c2 decimal(10,0) POSTHOOK: type: ALTERTABLE_RENAMECOL POSTHOOK: Input: default@alter_partition_change_col1 POSTHOOK: Output: default@alter_partition_change_col1 -PREHOOK: query: alter table alter_partition_change_col1 partition (p1='abc') change c2 c2 decimal(10,0) +PREHOOK: query: alter table alter_partition_change_col1 partition (p1='abc', p2='123') change c2 c2 decimal(10,0) PREHOOK: type: ALTERTABLE_RENAMECOL PREHOOK: Input: default@alter_partition_change_col1 -PREHOOK: Output: default@alter_partition_change_col1@p1=abc -POSTHOOK: query: alter table alter_partition_change_col1 partition (p1='abc') change c2 c2 decimal(10,0) +PREHOOK: Output: default@alter_partition_change_col1@p1=abc/p2=123 +POSTHOOK: query: alter table alter_partition_change_col1 partition (p1='abc', p2='123') change c2 c2 decimal(10,0) POSTHOOK: type: ALTERTABLE_RENAMECOL POSTHOOK: Input: default@alter_partition_change_col1 -POSTHOOK: Input: default@alter_partition_change_col1@p1=abc -POSTHOOK: Output: default@alter_partition_change_col1@p1=abc -PREHOOK: query: alter table alter_partition_change_col1 partition (p1='__HIVE_DEFAULT_PARTITION__') change c2 c2 decimal(10,0) +POSTHOOK: Input: default@alter_partition_change_col1@p1=abc/p2=123 +POSTHOOK: Output: default@alter_partition_change_col1@p1=abc/p2=123 +PREHOOK: query: alter table alter_partition_change_col1 partition (p1='__HIVE_DEFAULT_PARTITION__', p2='123') change c2 c2 decimal(10,0) PREHOOK: type: ALTERTABLE_RENAMECOL PREHOOK: Input: default@alter_partition_change_col1 -PREHOOK: Output: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__ -POSTHOOK: query: alter table alter_partition_change_col1 partition (p1='__HIVE_DEFAULT_PARTITION__') change c2 c2 decimal(10,0) +PREHOOK: Output: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__/p2=123 +POSTHOOK: query: alter table alter_partition_change_col1 partition (p1='__HIVE_DEFAULT_PARTITION__', p2='123') change c2 c2 decimal(10,0) POSTHOOK: type: ALTERTABLE_RENAMECOL POSTHOOK: Input: default@alter_partition_change_col1 -POSTHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__ -POSTHOOK: Output: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__ -PREHOOK: query: select * from alter_partition_change_col1 +POSTHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__/p2=123 +POSTHOOK: Output: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__/p2=123 +PREHOOK: query: select * from alter_partition_change_col1 where p1='abc' +PREHOOK: type: QUERY +PREHOOK: Input: default@alter_partition_change_col1 +PREHOOK: Input: default@alter_partition_change_col1@p1=abc/p2=123 +#### A masked pattern was here #### +POSTHOOK: query: select * from alter_partition_change_col1 where p1='abc' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alter_partition_change_col1 +POSTHOOK: Input: default@alter_partition_change_col1@p1=abc/p2=123 +#### A masked pattern was here #### +Beck 0 abc 123 +Beck 77 abc 123 +Beck 80 abc 123 +Cluck 6 abc 123 +Mary 33 abc 123 +Mary 4 abc 123 +Snow 56 abc 123 +Tom -12 abc 123 +Tom 19 abc 123 +Tom 235 abc 123 +PREHOOK: query: select * from alter_partition_change_col1 where p1='__HIVE_DEFAULT_PARTITION__' PREHOOK: type: QUERY PREHOOK: Input: default@alter_partition_change_col1 -PREHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__ -PREHOOK: Input: default@alter_partition_change_col1@p1=abc +PREHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__/p2=123 #### A masked pattern was here #### -POSTHOOK: query: select * from alter_partition_change_col1 +POSTHOOK: query: select * from alter_partition_change_col1 where p1='__HIVE_DEFAULT_PARTITION__' POSTHOOK: type: QUERY POSTHOOK: Input: default@alter_partition_change_col1 -POSTHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__ -POSTHOOK: Input: default@alter_partition_change_col1@p1=abc -#### A masked pattern was here #### -Beck 0 __HIVE_DEFAULT_PARTITION__ -Beck 0 abc -Beck 77 __HIVE_DEFAULT_PARTITION__ -Beck 77 abc -Beck 80 __HIVE_DEFAULT_PARTITION__ -Beck 80 abc -Cluck 6 __HIVE_DEFAULT_PARTITION__ -Cluck 6 abc -Mary 33 __HIVE_DEFAULT_PARTITION__ -Mary 33 abc -Mary 4 __HIVE_DEFAULT_PARTITION__ -Mary 4 abc -Snow 56 __HIVE_DEFAULT_PARTITION__ -Snow 56 abc -Tom -12 __HIVE_DEFAULT_PARTITION__ -Tom -12 abc -Tom 19 __HIVE_DEFAULT_PARTITION__ -Tom 19 abc -Tom 235 __HIVE_DEFAULT_PARTITION__ -Tom 235 abc +POSTHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__/p2=123 +#### A masked pattern was here #### +Beck 0 __HIVE_DEFAULT_PARTITION__ 123 +Beck 77 __HIVE_DEFAULT_PARTITION__ 123 +Beck 80 __HIVE_DEFAULT_PARTITION__ 123 +Cluck 6 __HIVE_DEFAULT_PARTITION__ 123 +Mary 33 __HIVE_DEFAULT_PARTITION__ 123 +Mary 4 __HIVE_DEFAULT_PARTITION__ 123 +Snow 56 __HIVE_DEFAULT_PARTITION__ 123 +Tom -12 __HIVE_DEFAULT_PARTITION__ 123 +Tom 19 __HIVE_DEFAULT_PARTITION__ 123 +Tom 235 __HIVE_DEFAULT_PARTITION__ 123 PREHOOK: query: -- Change the column type at the table level. Table-level describe shows the new type, but the existing partition does not. alter table alter_partition_change_col1 change c2 c2 decimal(14,4) PREHOOK: type: ALTERTABLE_RENAMECOL @@ -164,182 +180,214 @@ POSTHOOK: Input: default@alter_partition_change_col1 c1 string c2 decimal(14,4) p1 string +p2 string # Partition Information # col_name data_type comment p1 string -PREHOOK: query: describe alter_partition_change_col1 partition (p1='abc') +p2 string +PREHOOK: query: describe alter_partition_change_col1 partition (p1='abc', p2='123') PREHOOK: type: DESCTABLE PREHOOK: Input: default@alter_partition_change_col1 -POSTHOOK: query: describe alter_partition_change_col1 partition (p1='abc') +POSTHOOK: query: describe alter_partition_change_col1 partition (p1='abc', p2='123') POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@alter_partition_change_col1 c1 string c2 decimal(10,0) p1 string +p2 string # Partition Information # col_name data_type comment p1 string -PREHOOK: query: select * from alter_partition_change_col1 +p2 string +PREHOOK: query: select * from alter_partition_change_col1 where p1='abc' +PREHOOK: type: QUERY +PREHOOK: Input: default@alter_partition_change_col1 +PREHOOK: Input: default@alter_partition_change_col1@p1=abc/p2=123 +#### A masked pattern was here #### +POSTHOOK: query: select * from alter_partition_change_col1 where p1='abc' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alter_partition_change_col1 +POSTHOOK: Input: default@alter_partition_change_col1@p1=abc/p2=123 +#### A masked pattern was here #### +Beck 0 abc 123 +Beck 77 abc 123 +Beck 80 abc 123 +Cluck 6 abc 123 +Mary 33 abc 123 +Mary 4 abc 123 +Snow 56 abc 123 +Tom -12 abc 123 +Tom 19 abc 123 +Tom 235 abc 123 +PREHOOK: query: select * from alter_partition_change_col1 where p1='__HIVE_DEFAULT_PARTITION__' PREHOOK: type: QUERY PREHOOK: Input: default@alter_partition_change_col1 -PREHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__ -PREHOOK: Input: default@alter_partition_change_col1@p1=abc +PREHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__/p2=123 #### A masked pattern was here #### -POSTHOOK: query: select * from alter_partition_change_col1 +POSTHOOK: query: select * from alter_partition_change_col1 where p1='__HIVE_DEFAULT_PARTITION__' POSTHOOK: type: QUERY POSTHOOK: Input: default@alter_partition_change_col1 -POSTHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__ -POSTHOOK: Input: default@alter_partition_change_col1@p1=abc -#### A masked pattern was here #### -Beck 0 __HIVE_DEFAULT_PARTITION__ -Beck 0 abc -Beck 77 __HIVE_DEFAULT_PARTITION__ -Beck 77 abc -Beck 80 __HIVE_DEFAULT_PARTITION__ -Beck 80 abc -Cluck 6 __HIVE_DEFAULT_PARTITION__ -Cluck 6 abc -Mary 33 __HIVE_DEFAULT_PARTITION__ -Mary 33 abc -Mary 4 __HIVE_DEFAULT_PARTITION__ -Mary 4 abc -Snow 56 __HIVE_DEFAULT_PARTITION__ -Snow 56 abc -Tom -12 __HIVE_DEFAULT_PARTITION__ -Tom -12 abc -Tom 19 __HIVE_DEFAULT_PARTITION__ -Tom 19 abc -Tom 235 __HIVE_DEFAULT_PARTITION__ -Tom 235 abc +POSTHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__/p2=123 +#### A masked pattern was here #### +Beck 0 __HIVE_DEFAULT_PARTITION__ 123 +Beck 77 __HIVE_DEFAULT_PARTITION__ 123 +Beck 80 __HIVE_DEFAULT_PARTITION__ 123 +Cluck 6 __HIVE_DEFAULT_PARTITION__ 123 +Mary 33 __HIVE_DEFAULT_PARTITION__ 123 +Mary 4 __HIVE_DEFAULT_PARTITION__ 123 +Snow 56 __HIVE_DEFAULT_PARTITION__ 123 +Tom -12 __HIVE_DEFAULT_PARTITION__ 123 +Tom 19 __HIVE_DEFAULT_PARTITION__ 123 +Tom 235 __HIVE_DEFAULT_PARTITION__ 123 PREHOOK: query: -- now change the column type of the existing partition -alter table alter_partition_change_col1 partition (p1='abc') change c2 c2 decimal(14,4) +alter table alter_partition_change_col1 partition (p1='abc', p2='123') change c2 c2 decimal(14,4) PREHOOK: type: ALTERTABLE_RENAMECOL PREHOOK: Input: default@alter_partition_change_col1 -PREHOOK: Output: default@alter_partition_change_col1@p1=abc +PREHOOK: Output: default@alter_partition_change_col1@p1=abc/p2=123 POSTHOOK: query: -- now change the column type of the existing partition -alter table alter_partition_change_col1 partition (p1='abc') change c2 c2 decimal(14,4) +alter table alter_partition_change_col1 partition (p1='abc', p2='123') change c2 c2 decimal(14,4) POSTHOOK: type: ALTERTABLE_RENAMECOL POSTHOOK: Input: default@alter_partition_change_col1 -POSTHOOK: Input: default@alter_partition_change_col1@p1=abc -POSTHOOK: Output: default@alter_partition_change_col1@p1=abc -PREHOOK: query: describe alter_partition_change_col1 partition (p1='abc') +POSTHOOK: Input: default@alter_partition_change_col1@p1=abc/p2=123 +POSTHOOK: Output: default@alter_partition_change_col1@p1=abc/p2=123 +PREHOOK: query: describe alter_partition_change_col1 partition (p1='abc', p2='123') PREHOOK: type: DESCTABLE PREHOOK: Input: default@alter_partition_change_col1 -POSTHOOK: query: describe alter_partition_change_col1 partition (p1='abc') +POSTHOOK: query: describe alter_partition_change_col1 partition (p1='abc', p2='123') POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@alter_partition_change_col1 c1 string c2 decimal(14,4) p1 string +p2 string # Partition Information # col_name data_type comment p1 string -PREHOOK: query: select * from alter_partition_change_col1 +p2 string +PREHOOK: query: select * from alter_partition_change_col1 where p1='abc' +PREHOOK: type: QUERY +PREHOOK: Input: default@alter_partition_change_col1 +PREHOOK: Input: default@alter_partition_change_col1@p1=abc/p2=123 +#### A masked pattern was here #### +POSTHOOK: query: select * from alter_partition_change_col1 where p1='abc' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alter_partition_change_col1 +POSTHOOK: Input: default@alter_partition_change_col1@p1=abc/p2=123 +#### A masked pattern was here #### +Beck 0.0 abc 123 +Beck 77.341 abc 123 +Beck 79.9 abc 123 +Cluck 5.96 abc 123 +Mary 33.33 abc 123 +Mary 4.329 abc 123 +Snow 55.71 abc 123 +Tom -12.25 abc 123 +Tom 19.00 abc 123 +Tom 234.79 abc 123 +PREHOOK: query: select * from alter_partition_change_col1 where p1='__HIVE_DEFAULT_PARTITION__' PREHOOK: type: QUERY PREHOOK: Input: default@alter_partition_change_col1 -PREHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__ -PREHOOK: Input: default@alter_partition_change_col1@p1=abc +PREHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__/p2=123 #### A masked pattern was here #### -POSTHOOK: query: select * from alter_partition_change_col1 +POSTHOOK: query: select * from alter_partition_change_col1 where p1='__HIVE_DEFAULT_PARTITION__' POSTHOOK: type: QUERY POSTHOOK: Input: default@alter_partition_change_col1 -POSTHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__ -POSTHOOK: Input: default@alter_partition_change_col1@p1=abc -#### A masked pattern was here #### -Beck 0 __HIVE_DEFAULT_PARTITION__ -Beck 0.0 abc -Beck 77 __HIVE_DEFAULT_PARTITION__ -Beck 77.341 abc -Beck 79.9 abc -Beck 80 __HIVE_DEFAULT_PARTITION__ -Cluck 5.96 abc -Cluck 6 __HIVE_DEFAULT_PARTITION__ -Mary 33 __HIVE_DEFAULT_PARTITION__ -Mary 33.33 abc -Mary 4 __HIVE_DEFAULT_PARTITION__ -Mary 4.329 abc -Snow 55.71 abc -Snow 56 __HIVE_DEFAULT_PARTITION__ -Tom -12 __HIVE_DEFAULT_PARTITION__ -Tom -12.25 abc -Tom 19 __HIVE_DEFAULT_PARTITION__ -Tom 19.00 abc -Tom 234.79 abc -Tom 235 __HIVE_DEFAULT_PARTITION__ +POSTHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__/p2=123 +#### A masked pattern was here #### +Beck 0 __HIVE_DEFAULT_PARTITION__ 123 +Beck 77 __HIVE_DEFAULT_PARTITION__ 123 +Beck 80 __HIVE_DEFAULT_PARTITION__ 123 +Cluck 6 __HIVE_DEFAULT_PARTITION__ 123 +Mary 33 __HIVE_DEFAULT_PARTITION__ 123 +Mary 4 __HIVE_DEFAULT_PARTITION__ 123 +Snow 56 __HIVE_DEFAULT_PARTITION__ 123 +Tom -12 __HIVE_DEFAULT_PARTITION__ 123 +Tom 19 __HIVE_DEFAULT_PARTITION__ 123 +Tom 235 __HIVE_DEFAULT_PARTITION__ 123 PREHOOK: query: -- change column for default partition value -alter table alter_partition_change_col1 partition (p1='__HIVE_DEFAULT_PARTITION__') change c2 c2 decimal(14,4) +alter table alter_partition_change_col1 partition (p1='__HIVE_DEFAULT_PARTITION__', p2='123') change c2 c2 decimal(14,4) PREHOOK: type: ALTERTABLE_RENAMECOL PREHOOK: Input: default@alter_partition_change_col1 -PREHOOK: Output: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__ +PREHOOK: Output: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__/p2=123 POSTHOOK: query: -- change column for default partition value -alter table alter_partition_change_col1 partition (p1='__HIVE_DEFAULT_PARTITION__') change c2 c2 decimal(14,4) +alter table alter_partition_change_col1 partition (p1='__HIVE_DEFAULT_PARTITION__', p2='123') change c2 c2 decimal(14,4) POSTHOOK: type: ALTERTABLE_RENAMECOL POSTHOOK: Input: default@alter_partition_change_col1 -POSTHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__ -POSTHOOK: Output: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__ -PREHOOK: query: describe alter_partition_change_col1 partition (p1='__HIVE_DEFAULT_PARTITION__') +POSTHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__/p2=123 +POSTHOOK: Output: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__/p2=123 +PREHOOK: query: describe alter_partition_change_col1 partition (p1='__HIVE_DEFAULT_PARTITION__', p2='123') PREHOOK: type: DESCTABLE PREHOOK: Input: default@alter_partition_change_col1 -POSTHOOK: query: describe alter_partition_change_col1 partition (p1='__HIVE_DEFAULT_PARTITION__') +POSTHOOK: query: describe alter_partition_change_col1 partition (p1='__HIVE_DEFAULT_PARTITION__', p2='123') POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@alter_partition_change_col1 c1 string c2 decimal(14,4) p1 string +p2 string # Partition Information # col_name data_type comment p1 string -PREHOOK: query: select * from alter_partition_change_col1 +p2 string +PREHOOK: query: select * from alter_partition_change_col1 where p1='abc' PREHOOK: type: QUERY PREHOOK: Input: default@alter_partition_change_col1 -PREHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__ -PREHOOK: Input: default@alter_partition_change_col1@p1=abc +PREHOOK: Input: default@alter_partition_change_col1@p1=abc/p2=123 #### A masked pattern was here #### -POSTHOOK: query: select * from alter_partition_change_col1 +POSTHOOK: query: select * from alter_partition_change_col1 where p1='abc' POSTHOOK: type: QUERY POSTHOOK: Input: default@alter_partition_change_col1 -POSTHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__ -POSTHOOK: Input: default@alter_partition_change_col1@p1=abc -#### A masked pattern was here #### -Beck 0.0 __HIVE_DEFAULT_PARTITION__ -Beck 0.0 abc -Beck 77.341 __HIVE_DEFAULT_PARTITION__ -Beck 77.341 abc -Beck 79.9 __HIVE_DEFAULT_PARTITION__ -Beck 79.9 abc -Cluck 5.96 __HIVE_DEFAULT_PARTITION__ -Cluck 5.96 abc -Mary 33.33 __HIVE_DEFAULT_PARTITION__ -Mary 33.33 abc -Mary 4.329 __HIVE_DEFAULT_PARTITION__ -Mary 4.329 abc -Snow 55.71 __HIVE_DEFAULT_PARTITION__ -Snow 55.71 abc -Tom -12.25 __HIVE_DEFAULT_PARTITION__ -Tom -12.25 abc -Tom 19.00 __HIVE_DEFAULT_PARTITION__ -Tom 19.00 abc -Tom 234.79 __HIVE_DEFAULT_PARTITION__ -Tom 234.79 abc +POSTHOOK: Input: default@alter_partition_change_col1@p1=abc/p2=123 +#### A masked pattern was here #### +Beck 0.0 abc 123 +Beck 77.341 abc 123 +Beck 79.9 abc 123 +Cluck 5.96 abc 123 +Mary 33.33 abc 123 +Mary 4.329 abc 123 +Snow 55.71 abc 123 +Tom -12.25 abc 123 +Tom 19.00 abc 123 +Tom 234.79 abc 123 +PREHOOK: query: select * from alter_partition_change_col1 where p1='__HIVE_DEFAULT_PARTITION__' +PREHOOK: type: QUERY +PREHOOK: Input: default@alter_partition_change_col1 +PREHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__/p2=123 +#### A masked pattern was here #### +POSTHOOK: query: select * from alter_partition_change_col1 where p1='__HIVE_DEFAULT_PARTITION__' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alter_partition_change_col1 +POSTHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__/p2=123 +#### A masked pattern was here #### +Beck 0.0 __HIVE_DEFAULT_PARTITION__ 123 +Beck 77.341 __HIVE_DEFAULT_PARTITION__ 123 +Beck 79.9 __HIVE_DEFAULT_PARTITION__ 123 +Cluck 5.96 __HIVE_DEFAULT_PARTITION__ 123 +Mary 33.33 __HIVE_DEFAULT_PARTITION__ 123 +Mary 4.329 __HIVE_DEFAULT_PARTITION__ 123 +Snow 55.71 __HIVE_DEFAULT_PARTITION__ 123 +Tom -12.25 __HIVE_DEFAULT_PARTITION__ 123 +Tom 19.00 __HIVE_DEFAULT_PARTITION__ 123 +Tom 234.79 __HIVE_DEFAULT_PARTITION__ 123 PREHOOK: query: -- Try out replace columns -alter table alter_partition_change_col1 partition (p1='abc') replace columns (c1 string) +alter table alter_partition_change_col1 partition (p1='abc', p2='123') replace columns (c1 string) PREHOOK: type: ALTERTABLE_REPLACECOLS PREHOOK: Input: default@alter_partition_change_col1 -PREHOOK: Output: default@alter_partition_change_col1@p1=abc +PREHOOK: Output: default@alter_partition_change_col1@p1=abc/p2=123 POSTHOOK: query: -- Try out replace columns -alter table alter_partition_change_col1 partition (p1='abc') replace columns (c1 string) +alter table alter_partition_change_col1 partition (p1='abc', p2='123') replace columns (c1 string) POSTHOOK: type: ALTERTABLE_REPLACECOLS POSTHOOK: Input: default@alter_partition_change_col1 -POSTHOOK: Input: default@alter_partition_change_col1@p1=abc -POSTHOOK: Output: default@alter_partition_change_col1@p1=abc +POSTHOOK: Input: default@alter_partition_change_col1@p1=abc/p2=123 +POSTHOOK: Output: default@alter_partition_change_col1@p1=abc/p2=123 PREHOOK: query: describe alter_partition_change_col1 PREHOOK: type: DESCTABLE PREHOOK: Input: default@alter_partition_change_col1 @@ -349,56 +397,68 @@ POSTHOOK: Input: default@alter_partition_change_col1 c1 string c2 decimal(14,4) p1 string +p2 string # Partition Information # col_name data_type comment p1 string -PREHOOK: query: describe alter_partition_change_col1 partition (p1='abc') +p2 string +PREHOOK: query: describe alter_partition_change_col1 partition (p1='abc', p2='123') PREHOOK: type: DESCTABLE PREHOOK: Input: default@alter_partition_change_col1 -POSTHOOK: query: describe alter_partition_change_col1 partition (p1='abc') +POSTHOOK: query: describe alter_partition_change_col1 partition (p1='abc', p2='123') POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@alter_partition_change_col1 c1 string p1 string +p2 string # Partition Information # col_name data_type comment p1 string -PREHOOK: query: select * from alter_partition_change_col1 +p2 string +PREHOOK: query: select * from alter_partition_change_col1 where p1='abc' +PREHOOK: type: QUERY +PREHOOK: Input: default@alter_partition_change_col1 +PREHOOK: Input: default@alter_partition_change_col1@p1=abc/p2=123 +#### A masked pattern was here #### +POSTHOOK: query: select * from alter_partition_change_col1 where p1='abc' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alter_partition_change_col1 +POSTHOOK: Input: default@alter_partition_change_col1@p1=abc/p2=123 +#### A masked pattern was here #### +Beck NULL abc 123 +Beck NULL abc 123 +Beck NULL abc 123 +Cluck NULL abc 123 +Mary NULL abc 123 +Mary NULL abc 123 +Snow NULL abc 123 +Tom NULL abc 123 +Tom NULL abc 123 +Tom NULL abc 123 +PREHOOK: query: select * from alter_partition_change_col1 where p1='__HIVE_DEFAULT_PARTITION__' PREHOOK: type: QUERY PREHOOK: Input: default@alter_partition_change_col1 -PREHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__ -PREHOOK: Input: default@alter_partition_change_col1@p1=abc +PREHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__/p2=123 #### A masked pattern was here #### -POSTHOOK: query: select * from alter_partition_change_col1 +POSTHOOK: query: select * from alter_partition_change_col1 where p1='__HIVE_DEFAULT_PARTITION__' POSTHOOK: type: QUERY POSTHOOK: Input: default@alter_partition_change_col1 -POSTHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__ -POSTHOOK: Input: default@alter_partition_change_col1@p1=abc -#### A masked pattern was here #### -Beck 0.0 __HIVE_DEFAULT_PARTITION__ -Beck 77.341 __HIVE_DEFAULT_PARTITION__ -Beck 79.9 __HIVE_DEFAULT_PARTITION__ -Beck NULL abc -Beck NULL abc -Beck NULL abc -Cluck 5.96 __HIVE_DEFAULT_PARTITION__ -Cluck NULL abc -Mary 33.33 __HIVE_DEFAULT_PARTITION__ -Mary 4.329 __HIVE_DEFAULT_PARTITION__ -Mary NULL abc -Mary NULL abc -Snow 55.71 __HIVE_DEFAULT_PARTITION__ -Snow NULL abc -Tom -12.25 __HIVE_DEFAULT_PARTITION__ -Tom 19.00 __HIVE_DEFAULT_PARTITION__ -Tom 234.79 __HIVE_DEFAULT_PARTITION__ -Tom NULL abc -Tom NULL abc -Tom NULL abc +POSTHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__/p2=123 +#### A masked pattern was here #### +Beck 0.0 __HIVE_DEFAULT_PARTITION__ 123 +Beck 77.341 __HIVE_DEFAULT_PARTITION__ 123 +Beck 79.9 __HIVE_DEFAULT_PARTITION__ 123 +Cluck 5.96 __HIVE_DEFAULT_PARTITION__ 123 +Mary 33.33 __HIVE_DEFAULT_PARTITION__ 123 +Mary 4.329 __HIVE_DEFAULT_PARTITION__ 123 +Snow 55.71 __HIVE_DEFAULT_PARTITION__ 123 +Tom -12.25 __HIVE_DEFAULT_PARTITION__ 123 +Tom 19.00 __HIVE_DEFAULT_PARTITION__ 123 +Tom 234.79 __HIVE_DEFAULT_PARTITION__ 123 PREHOOK: query: alter table alter_partition_change_col1 replace columns (c1 string) PREHOOK: type: ALTERTABLE_REPLACECOLS PREHOOK: Input: default@alter_partition_change_col1 @@ -415,43 +475,53 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@alter_partition_change_col1 c1 string p1 string +p2 string # Partition Information # col_name data_type comment p1 string -PREHOOK: query: select * from alter_partition_change_col1 +p2 string +PREHOOK: query: select * from alter_partition_change_col1 where p1='abc' PREHOOK: type: QUERY PREHOOK: Input: default@alter_partition_change_col1 -PREHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__ -PREHOOK: Input: default@alter_partition_change_col1@p1=abc +PREHOOK: Input: default@alter_partition_change_col1@p1=abc/p2=123 #### A masked pattern was here #### -POSTHOOK: query: select * from alter_partition_change_col1 +POSTHOOK: query: select * from alter_partition_change_col1 where p1='abc' POSTHOOK: type: QUERY POSTHOOK: Input: default@alter_partition_change_col1 -POSTHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__ -POSTHOOK: Input: default@alter_partition_change_col1@p1=abc -#### A masked pattern was here #### -Beck __HIVE_DEFAULT_PARTITION__ -Beck __HIVE_DEFAULT_PARTITION__ -Beck __HIVE_DEFAULT_PARTITION__ -Beck abc -Beck abc -Beck abc -Cluck __HIVE_DEFAULT_PARTITION__ -Cluck abc -Mary __HIVE_DEFAULT_PARTITION__ -Mary __HIVE_DEFAULT_PARTITION__ -Mary abc -Mary abc -Snow __HIVE_DEFAULT_PARTITION__ -Snow abc -Tom __HIVE_DEFAULT_PARTITION__ -Tom __HIVE_DEFAULT_PARTITION__ -Tom __HIVE_DEFAULT_PARTITION__ -Tom abc -Tom abc -Tom abc +POSTHOOK: Input: default@alter_partition_change_col1@p1=abc/p2=123 +#### A masked pattern was here #### +Beck abc 123 +Beck abc 123 +Beck abc 123 +Cluck abc 123 +Mary abc 123 +Mary abc 123 +Snow abc 123 +Tom abc 123 +Tom abc 123 +Tom abc 123 +PREHOOK: query: select * from alter_partition_change_col1 where p1='__HIVE_DEFAULT_PARTITION__' +PREHOOK: type: QUERY +PREHOOK: Input: default@alter_partition_change_col1 +PREHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__/p2=123 +#### A masked pattern was here #### +POSTHOOK: query: select * from alter_partition_change_col1 where p1='__HIVE_DEFAULT_PARTITION__' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alter_partition_change_col1 +POSTHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__/p2=123 +#### A masked pattern was here #### +Beck __HIVE_DEFAULT_PARTITION__ 123 +Beck __HIVE_DEFAULT_PARTITION__ 123 +Beck __HIVE_DEFAULT_PARTITION__ 123 +Cluck __HIVE_DEFAULT_PARTITION__ 123 +Mary __HIVE_DEFAULT_PARTITION__ 123 +Mary __HIVE_DEFAULT_PARTITION__ 123 +Snow __HIVE_DEFAULT_PARTITION__ 123 +Tom __HIVE_DEFAULT_PARTITION__ 123 +Tom __HIVE_DEFAULT_PARTITION__ 123 +Tom __HIVE_DEFAULT_PARTITION__ 123 PREHOOK: query: -- Try add columns alter table alter_partition_change_col1 add columns (c2 decimal(14,4)) PREHOOK: type: ALTERTABLE_ADDCOLS @@ -471,108 +541,216 @@ POSTHOOK: Input: default@alter_partition_change_col1 c1 string c2 decimal(14,4) p1 string +p2 string # Partition Information # col_name data_type comment p1 string -PREHOOK: query: describe alter_partition_change_col1 partition (p1='abc') +p2 string +PREHOOK: query: describe alter_partition_change_col1 partition (p1='abc', p2='123') PREHOOK: type: DESCTABLE PREHOOK: Input: default@alter_partition_change_col1 -POSTHOOK: query: describe alter_partition_change_col1 partition (p1='abc') +POSTHOOK: query: describe alter_partition_change_col1 partition (p1='abc', p2='123') POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@alter_partition_change_col1 c1 string p1 string +p2 string # Partition Information # col_name data_type comment p1 string -PREHOOK: query: select * from alter_partition_change_col1 +p2 string +PREHOOK: query: select * from alter_partition_change_col1 where p1='abc' PREHOOK: type: QUERY PREHOOK: Input: default@alter_partition_change_col1 -PREHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__ -PREHOOK: Input: default@alter_partition_change_col1@p1=abc +PREHOOK: Input: default@alter_partition_change_col1@p1=abc/p2=123 #### A masked pattern was here #### -POSTHOOK: query: select * from alter_partition_change_col1 +POSTHOOK: query: select * from alter_partition_change_col1 where p1='abc' POSTHOOK: type: QUERY POSTHOOK: Input: default@alter_partition_change_col1 -POSTHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__ -POSTHOOK: Input: default@alter_partition_change_col1@p1=abc -#### A masked pattern was here #### -Beck 0.0 __HIVE_DEFAULT_PARTITION__ -Beck 77.341 __HIVE_DEFAULT_PARTITION__ -Beck 79.9 __HIVE_DEFAULT_PARTITION__ -Beck NULL abc -Beck NULL abc -Beck NULL abc -Cluck 5.96 __HIVE_DEFAULT_PARTITION__ -Cluck NULL abc -Mary 33.33 __HIVE_DEFAULT_PARTITION__ -Mary 4.329 __HIVE_DEFAULT_PARTITION__ -Mary NULL abc -Mary NULL abc -Snow 55.71 __HIVE_DEFAULT_PARTITION__ -Snow NULL abc -Tom -12.25 __HIVE_DEFAULT_PARTITION__ -Tom 19.00 __HIVE_DEFAULT_PARTITION__ -Tom 234.79 __HIVE_DEFAULT_PARTITION__ -Tom NULL abc -Tom NULL abc -Tom NULL abc -PREHOOK: query: alter table alter_partition_change_col1 partition (p1='abc') add columns (c2 decimal(14,4)) +POSTHOOK: Input: default@alter_partition_change_col1@p1=abc/p2=123 +#### A masked pattern was here #### +Beck NULL abc 123 +Beck NULL abc 123 +Beck NULL abc 123 +Cluck NULL abc 123 +Mary NULL abc 123 +Mary NULL abc 123 +Snow NULL abc 123 +Tom NULL abc 123 +Tom NULL abc 123 +Tom NULL abc 123 +PREHOOK: query: select * from alter_partition_change_col1 where p1='__HIVE_DEFAULT_PARTITION__' +PREHOOK: type: QUERY +PREHOOK: Input: default@alter_partition_change_col1 +PREHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__/p2=123 +#### A masked pattern was here #### +POSTHOOK: query: select * from alter_partition_change_col1 where p1='__HIVE_DEFAULT_PARTITION__' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alter_partition_change_col1 +POSTHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__/p2=123 +#### A masked pattern was here #### +Beck 0.0 __HIVE_DEFAULT_PARTITION__ 123 +Beck 77.341 __HIVE_DEFAULT_PARTITION__ 123 +Beck 79.9 __HIVE_DEFAULT_PARTITION__ 123 +Cluck 5.96 __HIVE_DEFAULT_PARTITION__ 123 +Mary 33.33 __HIVE_DEFAULT_PARTITION__ 123 +Mary 4.329 __HIVE_DEFAULT_PARTITION__ 123 +Snow 55.71 __HIVE_DEFAULT_PARTITION__ 123 +Tom -12.25 __HIVE_DEFAULT_PARTITION__ 123 +Tom 19.00 __HIVE_DEFAULT_PARTITION__ 123 +Tom 234.79 __HIVE_DEFAULT_PARTITION__ 123 +PREHOOK: query: alter table alter_partition_change_col1 partition (p1='abc', p2='123') add columns (c2 decimal(14,4)) PREHOOK: type: ALTERTABLE_ADDCOLS PREHOOK: Input: default@alter_partition_change_col1 -PREHOOK: Output: default@alter_partition_change_col1@p1=abc -POSTHOOK: query: alter table alter_partition_change_col1 partition (p1='abc') add columns (c2 decimal(14,4)) +PREHOOK: Output: default@alter_partition_change_col1@p1=abc/p2=123 +POSTHOOK: query: alter table alter_partition_change_col1 partition (p1='abc', p2='123') add columns (c2 decimal(14,4)) POSTHOOK: type: ALTERTABLE_ADDCOLS POSTHOOK: Input: default@alter_partition_change_col1 -POSTHOOK: Input: default@alter_partition_change_col1@p1=abc -POSTHOOK: Output: default@alter_partition_change_col1@p1=abc -PREHOOK: query: describe alter_partition_change_col1 partition (p1='abc') +POSTHOOK: Input: default@alter_partition_change_col1@p1=abc/p2=123 +POSTHOOK: Output: default@alter_partition_change_col1@p1=abc/p2=123 +PREHOOK: query: describe alter_partition_change_col1 partition (p1='abc', p2='123') PREHOOK: type: DESCTABLE PREHOOK: Input: default@alter_partition_change_col1 -POSTHOOK: query: describe alter_partition_change_col1 partition (p1='abc') +POSTHOOK: query: describe alter_partition_change_col1 partition (p1='abc', p2='123') POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@alter_partition_change_col1 c1 string c2 decimal(14,4) p1 string +p2 string # Partition Information # col_name data_type comment p1 string -PREHOOK: query: select * from alter_partition_change_col1 +p2 string +PREHOOK: query: select * from alter_partition_change_col1 where p1='abc' PREHOOK: type: QUERY PREHOOK: Input: default@alter_partition_change_col1 -PREHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__ -PREHOOK: Input: default@alter_partition_change_col1@p1=abc +PREHOOK: Input: default@alter_partition_change_col1@p1=abc/p2=123 #### A masked pattern was here #### -POSTHOOK: query: select * from alter_partition_change_col1 +POSTHOOK: query: select * from alter_partition_change_col1 where p1='abc' POSTHOOK: type: QUERY POSTHOOK: Input: default@alter_partition_change_col1 -POSTHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__ -POSTHOOK: Input: default@alter_partition_change_col1@p1=abc -#### A masked pattern was here #### -Beck 0.0 __HIVE_DEFAULT_PARTITION__ -Beck 0.0 abc -Beck 77.341 __HIVE_DEFAULT_PARTITION__ -Beck 77.341 abc -Beck 79.9 __HIVE_DEFAULT_PARTITION__ -Beck 79.9 abc -Cluck 5.96 __HIVE_DEFAULT_PARTITION__ -Cluck 5.96 abc -Mary 33.33 __HIVE_DEFAULT_PARTITION__ -Mary 33.33 abc -Mary 4.329 __HIVE_DEFAULT_PARTITION__ -Mary 4.329 abc -Snow 55.71 __HIVE_DEFAULT_PARTITION__ -Snow 55.71 abc -Tom -12.25 __HIVE_DEFAULT_PARTITION__ -Tom -12.25 abc -Tom 19.00 __HIVE_DEFAULT_PARTITION__ -Tom 19.00 abc -Tom 234.79 __HIVE_DEFAULT_PARTITION__ -Tom 234.79 abc +POSTHOOK: Input: default@alter_partition_change_col1@p1=abc/p2=123 +#### A masked pattern was here #### +Beck 0.0 abc 123 +Beck 77.341 abc 123 +Beck 79.9 abc 123 +Cluck 5.96 abc 123 +Mary 33.33 abc 123 +Mary 4.329 abc 123 +Snow 55.71 abc 123 +Tom -12.25 abc 123 +Tom 19.00 abc 123 +Tom 234.79 abc 123 +PREHOOK: query: select * from alter_partition_change_col1 where p1='__HIVE_DEFAULT_PARTITION__' +PREHOOK: type: QUERY +PREHOOK: Input: default@alter_partition_change_col1 +PREHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__/p2=123 +#### A masked pattern was here #### +POSTHOOK: query: select * from alter_partition_change_col1 where p1='__HIVE_DEFAULT_PARTITION__' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alter_partition_change_col1 +POSTHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__/p2=123 +#### A masked pattern was here #### +Beck 0.0 __HIVE_DEFAULT_PARTITION__ 123 +Beck 77.341 __HIVE_DEFAULT_PARTITION__ 123 +Beck 79.9 __HIVE_DEFAULT_PARTITION__ 123 +Cluck 5.96 __HIVE_DEFAULT_PARTITION__ 123 +Mary 33.33 __HIVE_DEFAULT_PARTITION__ 123 +Mary 4.329 __HIVE_DEFAULT_PARTITION__ 123 +Snow 55.71 __HIVE_DEFAULT_PARTITION__ 123 +Tom -12.25 __HIVE_DEFAULT_PARTITION__ 123 +Tom 19.00 __HIVE_DEFAULT_PARTITION__ 123 +Tom 234.79 __HIVE_DEFAULT_PARTITION__ 123 +PREHOOK: query: -- Try changing column for all partitions at once +alter table alter_partition_change_col1 partition (p1, p2='123') change column c2 c2 decimal(10,0) +PREHOOK: type: ALTERTABLE_RENAMECOL +PREHOOK: Input: default@alter_partition_change_col1 +PREHOOK: Output: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__/p2=123 +PREHOOK: Output: default@alter_partition_change_col1@p1=abc/p2=123 +POSTHOOK: query: -- Try changing column for all partitions at once +alter table alter_partition_change_col1 partition (p1, p2='123') change column c2 c2 decimal(10,0) +POSTHOOK: type: ALTERTABLE_RENAMECOL +POSTHOOK: Input: default@alter_partition_change_col1 +POSTHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__/p2=123 +POSTHOOK: Input: default@alter_partition_change_col1@p1=abc/p2=123 +POSTHOOK: Output: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__/p2=123 +POSTHOOK: Output: default@alter_partition_change_col1@p1=abc/p2=123 +PREHOOK: query: describe alter_partition_change_col1 partition (p1='abc', p2='123') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@alter_partition_change_col1 +POSTHOOK: query: describe alter_partition_change_col1 partition (p1='abc', p2='123') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@alter_partition_change_col1 +c1 string +c2 decimal(10,0) +p1 string +p2 string + +# Partition Information +# col_name data_type comment + +p1 string +p2 string +PREHOOK: query: describe alter_partition_change_col1 partition (p1='__HIVE_DEFAULT_PARTITION__', p2='123') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@alter_partition_change_col1 +POSTHOOK: query: describe alter_partition_change_col1 partition (p1='__HIVE_DEFAULT_PARTITION__', p2='123') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@alter_partition_change_col1 +c1 string +c2 decimal(10,0) +p1 string +p2 string + +# Partition Information +# col_name data_type comment + +p1 string +p2 string +PREHOOK: query: select * from alter_partition_change_col1 where p1='abc' +PREHOOK: type: QUERY +PREHOOK: Input: default@alter_partition_change_col1 +PREHOOK: Input: default@alter_partition_change_col1@p1=abc/p2=123 +#### A masked pattern was here #### +POSTHOOK: query: select * from alter_partition_change_col1 where p1='abc' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alter_partition_change_col1 +POSTHOOK: Input: default@alter_partition_change_col1@p1=abc/p2=123 +#### A masked pattern was here #### +Beck 0 abc 123 +Beck 77 abc 123 +Beck 80 abc 123 +Cluck 6 abc 123 +Mary 33 abc 123 +Mary 4 abc 123 +Snow 56 abc 123 +Tom -12 abc 123 +Tom 19 abc 123 +Tom 235 abc 123 +PREHOOK: query: select * from alter_partition_change_col1 where p1='__HIVE_DEFAULT_PARTITION__' +PREHOOK: type: QUERY +PREHOOK: Input: default@alter_partition_change_col1 +PREHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__/p2=123 +#### A masked pattern was here #### +POSTHOOK: query: select * from alter_partition_change_col1 where p1='__HIVE_DEFAULT_PARTITION__' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alter_partition_change_col1 +POSTHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__/p2=123 +#### A masked pattern was here #### +Beck 0 __HIVE_DEFAULT_PARTITION__ 123 +Beck 77 __HIVE_DEFAULT_PARTITION__ 123 +Beck 80 __HIVE_DEFAULT_PARTITION__ 123 +Cluck 6 __HIVE_DEFAULT_PARTITION__ 123 +Mary 33 __HIVE_DEFAULT_PARTITION__ 123 +Mary 4 __HIVE_DEFAULT_PARTITION__ 123 +Snow 56 __HIVE_DEFAULT_PARTITION__ 123 +Tom -12 __HIVE_DEFAULT_PARTITION__ 123 +Tom 19 __HIVE_DEFAULT_PARTITION__ 123 +Tom 235 __HIVE_DEFAULT_PARTITION__ 123 diff --git a/ql/src/test/results/clientpositive/ambiguous_col.q.out b/ql/src/test/results/clientpositive/ambiguous_col.q.out index e8760f1..d583162 100644 --- a/ql/src/test/results/clientpositive/ambiguous_col.q.out +++ b/ql/src/test/results/clientpositive/ambiguous_col.q.out @@ -53,8 +53,8 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 + expressions: _col0 (type: string), _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -124,8 +124,8 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 + expressions: _col0 (type: string), _col0 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -195,8 +195,8 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 + expressions: _col0 (type: string), _col0 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/ql/src/test/results/clientpositive/annotate_stats_groupby.q.out b/ql/src/test/results/clientpositive/annotate_stats_groupby.q.out index 1459b44..41a0083 100644 --- a/ql/src/test/results/clientpositive/annotate_stats_groupby.q.out +++ b/ql/src/test/results/clientpositive/annotate_stats_groupby.q.out @@ -177,17 +177,17 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 400 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 8 Data size: 800 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 400 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 8 Data size: 800 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: min(_col1) keys: _col0 (type: string), _col2 (type: bigint) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 416 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 8 Data size: 832 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false table: @@ -203,7 +203,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 4 Data size: 416 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 8 Data size: 832 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col2 (type: int) Reduce Operator Tree: Group By Operator @@ -211,14 +211,14 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 8 Data size: 832 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 8 Data size: 832 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 8 Data size: 832 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -852,14 +852,14 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 344 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 344 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4 Data size: 344 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/annotate_stats_groupby2.q.out b/ql/src/test/results/clientpositive/annotate_stats_groupby2.q.out index f991191..2f85c92 100644 --- a/ql/src/test/results/clientpositive/annotate_stats_groupby2.q.out +++ b/ql/src/test/results/clientpositive/annotate_stats_groupby2.q.out @@ -274,25 +274,25 @@ STAGE PLANS: keys: state (type: string), votes (type: bigint) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 860 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 8 Data size: 688 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 10 Data size: 860 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 8 Data size: 688 Basic stats: COMPLETE Column stats: PARTIAL Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 430 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 430 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 5 Data size: 430 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out b/ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out new file mode 100644 index 0000000..ea800a2 --- /dev/null +++ b/ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out @@ -0,0 +1,987 @@ +PREHOOK: query: drop table store_sales +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table store_sales +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table store +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table store +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table customer_address +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table customer_address +POSTHOOK: type: DROPTABLE +PREHOOK: query: -- s_store_sk is PK, ss_store_sk is FK +-- ca_address_sk is PK, ss_addr_sk is FK + +create table store_sales +( + ss_sold_date_sk int, + ss_sold_time_sk int, + ss_item_sk int, + ss_customer_sk int, + ss_cdemo_sk int, + ss_hdemo_sk int, + ss_addr_sk int, + ss_store_sk int, + ss_promo_sk int, + ss_ticket_number int, + ss_quantity int, + ss_wholesale_cost float, + ss_list_price float, + ss_sales_price float, + ss_ext_discount_amt float, + ss_ext_sales_price float, + ss_ext_wholesale_cost float, + ss_ext_list_price float, + ss_ext_tax float, + ss_coupon_amt float, + ss_net_paid float, + ss_net_paid_inc_tax float, + ss_net_profit float +) +row format delimited fields terminated by '|' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@store_sales +POSTHOOK: query: -- s_store_sk is PK, ss_store_sk is FK +-- ca_address_sk is PK, ss_addr_sk is FK + +create table store_sales +( + ss_sold_date_sk int, + ss_sold_time_sk int, + ss_item_sk int, + ss_customer_sk int, + ss_cdemo_sk int, + ss_hdemo_sk int, + ss_addr_sk int, + ss_store_sk int, + ss_promo_sk int, + ss_ticket_number int, + ss_quantity int, + ss_wholesale_cost float, + ss_list_price float, + ss_sales_price float, + ss_ext_discount_amt float, + ss_ext_sales_price float, + ss_ext_wholesale_cost float, + ss_ext_list_price float, + ss_ext_tax float, + ss_coupon_amt float, + ss_net_paid float, + ss_net_paid_inc_tax float, + ss_net_profit float +) +row format delimited fields terminated by '|' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@store_sales +PREHOOK: query: create table store +( + s_store_sk int, + s_store_id string, + s_rec_start_date string, + s_rec_end_date string, + s_closed_date_sk int, + s_store_name string, + s_number_employees int, + s_floor_space int, + s_hours string, + s_manager string, + s_market_id int, + s_geography_class string, + s_market_desc string, + s_market_manager string, + s_division_id int, + s_division_name string, + s_company_id int, + s_company_name string, + s_street_number string, + s_street_name string, + s_street_type string, + s_suite_number string, + s_city string, + s_county string, + s_state string, + s_zip string, + s_country string, + s_gmt_offset float, + s_tax_precentage float +) +row format delimited fields terminated by '|' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@store +POSTHOOK: query: create table store +( + s_store_sk int, + s_store_id string, + s_rec_start_date string, + s_rec_end_date string, + s_closed_date_sk int, + s_store_name string, + s_number_employees int, + s_floor_space int, + s_hours string, + s_manager string, + s_market_id int, + s_geography_class string, + s_market_desc string, + s_market_manager string, + s_division_id int, + s_division_name string, + s_company_id int, + s_company_name string, + s_street_number string, + s_street_name string, + s_street_type string, + s_suite_number string, + s_city string, + s_county string, + s_state string, + s_zip string, + s_country string, + s_gmt_offset float, + s_tax_precentage float +) +row format delimited fields terminated by '|' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@store +PREHOOK: query: create table customer_address +( + ca_address_sk int, + ca_address_id string, + ca_street_number string, + ca_street_name string, + ca_street_type string, + ca_suite_number string, + ca_city string, + ca_county string, + ca_state string, + ca_zip string, + ca_country string, + ca_gmt_offset float, + ca_location_type string +) +row format delimited fields terminated by '|' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@customer_address +POSTHOOK: query: create table customer_address +( + ca_address_sk int, + ca_address_id string, + ca_street_number string, + ca_street_name string, + ca_street_type string, + ca_suite_number string, + ca_city string, + ca_county string, + ca_state string, + ca_zip string, + ca_country string, + ca_gmt_offset float, + ca_location_type string +) +row format delimited fields terminated by '|' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@customer_address +PREHOOK: query: load data local inpath '../../data/files/store.txt' overwrite into table store +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@store +POSTHOOK: query: load data local inpath '../../data/files/store.txt' overwrite into table store +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@store +PREHOOK: query: load data local inpath '../../data/files/store_sales.txt' overwrite into table store_sales +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@store_sales +POSTHOOK: query: load data local inpath '../../data/files/store_sales.txt' overwrite into table store_sales +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@store_sales +PREHOOK: query: load data local inpath '../../data/files/customer_address.txt' overwrite into table customer_address +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@customer_address +POSTHOOK: query: load data local inpath '../../data/files/customer_address.txt' overwrite into table customer_address +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@customer_address +PREHOOK: query: analyze table store compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@store +PREHOOK: Output: default@store +POSTHOOK: query: analyze table store compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@store +POSTHOOK: Output: default@store +PREHOOK: query: analyze table store compute statistics for columns s_store_sk, s_floor_space +PREHOOK: type: QUERY +PREHOOK: Input: default@store +#### A masked pattern was here #### +POSTHOOK: query: analyze table store compute statistics for columns s_store_sk, s_floor_space +POSTHOOK: type: QUERY +POSTHOOK: Input: default@store +#### A masked pattern was here #### +PREHOOK: query: analyze table store_sales compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@store_sales +PREHOOK: Output: default@store_sales +POSTHOOK: query: analyze table store_sales compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@store_sales +POSTHOOK: Output: default@store_sales +PREHOOK: query: analyze table store_sales compute statistics for columns ss_store_sk, ss_addr_sk, ss_quantity +PREHOOK: type: QUERY +PREHOOK: Input: default@store_sales +#### A masked pattern was here #### +POSTHOOK: query: analyze table store_sales compute statistics for columns ss_store_sk, ss_addr_sk, ss_quantity +POSTHOOK: type: QUERY +POSTHOOK: Input: default@store_sales +#### A masked pattern was here #### +PREHOOK: query: analyze table customer_address compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@customer_address +PREHOOK: Output: default@customer_address +POSTHOOK: query: analyze table customer_address compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@customer_address +POSTHOOK: Output: default@customer_address +PREHOOK: query: analyze table customer_address compute statistics for columns ca_address_sk +PREHOOK: type: QUERY +PREHOOK: Input: default@customer_address +#### A masked pattern was here #### +POSTHOOK: query: analyze table customer_address compute statistics for columns ca_address_sk +POSTHOOK: type: QUERY +POSTHOOK: Input: default@customer_address +#### A masked pattern was here #### +PREHOOK: query: explain select s.s_store_sk from store s join store_sales ss on (s.s_store_sk = ss.ss_store_sk) +PREHOOK: type: QUERY +POSTHOOK: query: explain select s.s_store_sk from store s join store_sales ss on (s.s_store_sk = ss.ss_store_sk) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 12 Data size: 3143 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: s_store_sk is not null (type: boolean) + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: s_store_sk (type: int) + sort order: + + Map-reduce partition columns: s_store_sk (type: int) + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + TableScan + alias: ss + Statistics: Num rows: 1000 Data size: 130523 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ss_store_sk is not null (type: boolean) + Statistics: Num rows: 964 Data size: 3716 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: ss_store_sk (type: int) + sort order: + + Map-reduce partition columns: ss_store_sk (type: int) + Statistics: Num rows: 964 Data size: 3716 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {KEY.reducesinkkey0} + 1 + outputColumnNames: _col0 + Statistics: Num rows: 964 Data size: 3856 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 964 Data size: 3856 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 964 Data size: 3856 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select s.s_store_sk from store s join store_sales ss on (s.s_store_sk = ss.ss_store_sk) where s.s_store_sk > 0 +PREHOOK: type: QUERY +POSTHOOK: query: explain select s.s_store_sk from store s join store_sales ss on (s.s_store_sk = ss.ss_store_sk) where s.s_store_sk > 0 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 12 Data size: 3143 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (s_store_sk > 0) (type: boolean) + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: s_store_sk (type: int) + sort order: + + Map-reduce partition columns: s_store_sk (type: int) + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + TableScan + alias: ss + Statistics: Num rows: 1000 Data size: 130523 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (ss_store_sk > 0) (type: boolean) + Statistics: Num rows: 333 Data size: 1284 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: ss_store_sk (type: int) + sort order: + + Map-reduce partition columns: ss_store_sk (type: int) + Statistics: Num rows: 333 Data size: 1284 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {KEY.reducesinkkey0} + 1 + outputColumnNames: _col0 + Statistics: Num rows: 111 Data size: 444 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 111 Data size: 444 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 111 Data size: 444 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select s.s_store_sk from store s join store_sales ss on (s.s_store_sk = ss.ss_store_sk) where s.s_company_id > 0 and ss.ss_quantity > 10 +PREHOOK: type: QUERY +POSTHOOK: query: explain select s.s_store_sk from store s join store_sales ss on (s.s_store_sk = ss.ss_store_sk) where s.s_company_id > 0 and ss.ss_quantity > 10 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 12 Data size: 3143 Basic stats: COMPLETE Column stats: PARTIAL + Filter Operator + predicate: (s_store_sk is not null and (s_company_id > 0)) (type: boolean) + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: s_store_sk (type: int) + sort order: + + Map-reduce partition columns: s_store_sk (type: int) + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL + TableScan + alias: ss + Statistics: Num rows: 1000 Data size: 130523 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (ss_store_sk is not null and (ss_quantity > 10)) (type: boolean) + Statistics: Num rows: 321 Data size: 2460 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: ss_store_sk (type: int) + sort order: + + Map-reduce partition columns: ss_store_sk (type: int) + Statistics: Num rows: 321 Data size: 2460 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {KEY.reducesinkkey0} + 1 + outputColumnNames: _col0 + Statistics: Num rows: 107 Data size: 428 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 107 Data size: 428 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 107 Data size: 428 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select s.s_store_sk from store s join store_sales ss on (s.s_store_sk = ss.ss_store_sk) where s.s_floor_space > 0 +PREHOOK: type: QUERY +POSTHOOK: query: explain select s.s_store_sk from store s join store_sales ss on (s.s_store_sk = ss.ss_store_sk) where s.s_floor_space > 0 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 12 Data size: 3143 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (s_store_sk is not null and (s_floor_space > 0)) (type: boolean) + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: s_store_sk (type: int) + sort order: + + Map-reduce partition columns: s_store_sk (type: int) + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + TableScan + alias: ss + Statistics: Num rows: 1000 Data size: 130523 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ss_store_sk is not null (type: boolean) + Statistics: Num rows: 964 Data size: 3716 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: ss_store_sk (type: int) + sort order: + + Map-reduce partition columns: ss_store_sk (type: int) + Statistics: Num rows: 964 Data size: 3716 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {KEY.reducesinkkey0} + 1 + outputColumnNames: _col0 + Statistics: Num rows: 322 Data size: 1288 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 322 Data size: 1288 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 322 Data size: 1288 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select s.s_store_sk from store s join store_sales ss on (s.s_store_sk = ss.ss_store_sk) where ss.ss_quantity > 10 +PREHOOK: type: QUERY +POSTHOOK: query: explain select s.s_store_sk from store s join store_sales ss on (s.s_store_sk = ss.ss_store_sk) where ss.ss_quantity > 10 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 12 Data size: 3143 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: s_store_sk is not null (type: boolean) + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: s_store_sk (type: int) + sort order: + + Map-reduce partition columns: s_store_sk (type: int) + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + TableScan + alias: ss + Statistics: Num rows: 1000 Data size: 130523 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (ss_store_sk is not null and (ss_quantity > 10)) (type: boolean) + Statistics: Num rows: 321 Data size: 2460 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: ss_store_sk (type: int) + sort order: + + Map-reduce partition columns: ss_store_sk (type: int) + Statistics: Num rows: 321 Data size: 2460 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {KEY.reducesinkkey0} + 1 + outputColumnNames: _col0 + Statistics: Num rows: 321 Data size: 1284 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 321 Data size: 1284 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 321 Data size: 1284 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select s.s_store_sk from store s join store_sales ss on (s.s_store_sk = ss.ss_store_sk) join store s1 on (s1.s_store_sk = ss.ss_store_sk) +PREHOOK: type: QUERY +POSTHOOK: query: explain select s.s_store_sk from store s join store_sales ss on (s.s_store_sk = ss.ss_store_sk) join store s1 on (s1.s_store_sk = ss.ss_store_sk) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 12 Data size: 3143 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: s_store_sk is not null (type: boolean) + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: s_store_sk (type: int) + sort order: + + Map-reduce partition columns: s_store_sk (type: int) + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + TableScan + alias: s + Statistics: Num rows: 12 Data size: 3143 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: s_store_sk is not null (type: boolean) + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: s_store_sk (type: int) + sort order: + + Map-reduce partition columns: s_store_sk (type: int) + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + TableScan + alias: ss + Statistics: Num rows: 1000 Data size: 130523 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ss_store_sk is not null (type: boolean) + Statistics: Num rows: 964 Data size: 3716 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: ss_store_sk (type: int) + sort order: + + Map-reduce partition columns: ss_store_sk (type: int) + Statistics: Num rows: 964 Data size: 3716 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + condition expressions: + 0 {KEY.reducesinkkey0} + 1 + 2 + outputColumnNames: _col0 + Statistics: Num rows: 964 Data size: 3856 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 964 Data size: 3856 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 964 Data size: 3856 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select s.s_store_sk from store s join store_sales ss on (s.s_store_sk = ss.ss_store_sk) join store s1 on (s1.s_store_sk = ss.ss_store_sk) where s.s_store_sk > 1000 +PREHOOK: type: QUERY +POSTHOOK: query: explain select s.s_store_sk from store s join store_sales ss on (s.s_store_sk = ss.ss_store_sk) join store s1 on (s1.s_store_sk = ss.ss_store_sk) where s.s_store_sk > 1000 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 12 Data size: 3143 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (s_store_sk > 1000) (type: boolean) + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: s_store_sk (type: int) + sort order: + + Map-reduce partition columns: s_store_sk (type: int) + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + TableScan + alias: s + Statistics: Num rows: 12 Data size: 3143 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (s_store_sk > 1000) (type: boolean) + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: s_store_sk (type: int) + sort order: + + Map-reduce partition columns: s_store_sk (type: int) + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + TableScan + alias: ss + Statistics: Num rows: 1000 Data size: 130523 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (ss_store_sk > 1000) (type: boolean) + Statistics: Num rows: 333 Data size: 1284 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: ss_store_sk (type: int) + sort order: + + Map-reduce partition columns: ss_store_sk (type: int) + Statistics: Num rows: 333 Data size: 1284 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + condition expressions: + 0 {KEY.reducesinkkey0} + 1 + 2 + outputColumnNames: _col0 + Statistics: Num rows: 38 Data size: 152 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 38 Data size: 152 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 38 Data size: 152 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select s.s_store_sk from store s join store_sales ss on (s.s_store_sk = ss.ss_store_sk) join store s1 on (s1.s_store_sk = ss.ss_store_sk) where s.s_floor_space > 1000 +PREHOOK: type: QUERY +POSTHOOK: query: explain select s.s_store_sk from store s join store_sales ss on (s.s_store_sk = ss.ss_store_sk) join store s1 on (s1.s_store_sk = ss.ss_store_sk) where s.s_floor_space > 1000 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 12 Data size: 3143 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: s_store_sk is not null (type: boolean) + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: s_store_sk (type: int) + sort order: + + Map-reduce partition columns: s_store_sk (type: int) + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + TableScan + alias: s + Statistics: Num rows: 12 Data size: 3143 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (s_store_sk is not null and (s_floor_space > 1000)) (type: boolean) + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: s_store_sk (type: int) + sort order: + + Map-reduce partition columns: s_store_sk (type: int) + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + TableScan + alias: ss + Statistics: Num rows: 1000 Data size: 130523 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ss_store_sk is not null (type: boolean) + Statistics: Num rows: 964 Data size: 3716 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: ss_store_sk (type: int) + sort order: + + Map-reduce partition columns: ss_store_sk (type: int) + Statistics: Num rows: 964 Data size: 3716 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + condition expressions: + 0 {KEY.reducesinkkey0} + 1 + 2 + outputColumnNames: _col0 + Statistics: Num rows: 322 Data size: 1288 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 322 Data size: 1288 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 322 Data size: 1288 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select s.s_store_sk from store s join store_sales ss on (s.s_store_sk = ss.ss_store_sk) join store s1 on (s1.s_store_sk = ss.ss_store_sk) where ss.ss_quantity > 10 +PREHOOK: type: QUERY +POSTHOOK: query: explain select s.s_store_sk from store s join store_sales ss on (s.s_store_sk = ss.ss_store_sk) join store s1 on (s1.s_store_sk = ss.ss_store_sk) where ss.ss_quantity > 10 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 12 Data size: 3143 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: s_store_sk is not null (type: boolean) + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: s_store_sk (type: int) + sort order: + + Map-reduce partition columns: s_store_sk (type: int) + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + TableScan + alias: s + Statistics: Num rows: 12 Data size: 3143 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: s_store_sk is not null (type: boolean) + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: s_store_sk (type: int) + sort order: + + Map-reduce partition columns: s_store_sk (type: int) + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + TableScan + alias: ss + Statistics: Num rows: 1000 Data size: 130523 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (ss_store_sk is not null and (ss_quantity > 10)) (type: boolean) + Statistics: Num rows: 321 Data size: 2460 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: ss_store_sk (type: int) + sort order: + + Map-reduce partition columns: ss_store_sk (type: int) + Statistics: Num rows: 321 Data size: 2460 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + condition expressions: + 0 {KEY.reducesinkkey0} + 1 + 2 + outputColumnNames: _col0 + Statistics: Num rows: 321 Data size: 1284 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 321 Data size: 1284 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 321 Data size: 1284 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select s.s_store_sk from store s join store_sales ss on (s.s_store_sk = ss.ss_store_sk) join customer_address ca on (ca.ca_address_sk = ss.ss_addr_sk) +PREHOOK: type: QUERY +POSTHOOK: query: explain select s.s_store_sk from store s join store_sales ss on (s.s_store_sk = ss.ss_store_sk) join customer_address ca on (ca.ca_address_sk = ss.ss_addr_sk) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-1 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 12 Data size: 3143 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: s_store_sk is not null (type: boolean) + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: s_store_sk (type: int) + sort order: + + Map-reduce partition columns: s_store_sk (type: int) + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + TableScan + alias: ss + Statistics: Num rows: 1000 Data size: 130523 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (ss_store_sk is not null and ss_addr_sk is not null) (type: boolean) + Statistics: Num rows: 916 Data size: 7012 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: ss_store_sk (type: int) + sort order: + + Map-reduce partition columns: ss_store_sk (type: int) + Statistics: Num rows: 916 Data size: 7012 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: ss_addr_sk (type: int) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {KEY.reducesinkkey0} + 1 {VALUE._col6} + outputColumnNames: _col0, _col38 + Statistics: Num rows: 916 Data size: 7328 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: ca + Statistics: Num rows: 20 Data size: 2114 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ca_address_sk is not null (type: boolean) + Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: ca_address_sk (type: int) + sort order: + + Map-reduce partition columns: ca_address_sk (type: int) + Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + TableScan + Reduce Output Operator + key expressions: _col38 (type: int) + sort order: + + Map-reduce partition columns: _col38 (type: int) + Statistics: Num rows: 916 Data size: 7328 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {VALUE._col0} + 1 + outputColumnNames: _col0 + Statistics: Num rows: 210 Data size: 840 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 210 Data size: 840 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 210 Data size: 840 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: drop table store_sales +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@store_sales +PREHOOK: Output: default@store_sales +POSTHOOK: query: drop table store_sales +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@store_sales +POSTHOOK: Output: default@store_sales +PREHOOK: query: drop table store +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@store +PREHOOK: Output: default@store +POSTHOOK: query: drop table store +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@store +POSTHOOK: Output: default@store +PREHOOK: query: drop table customer_address +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@customer_address +PREHOOK: Output: default@customer_address +POSTHOOK: query: drop table customer_address +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@customer_address +POSTHOOK: Output: default@customer_address diff --git a/ql/src/test/results/clientpositive/annotate_stats_part.q.out b/ql/src/test/results/clientpositive/annotate_stats_part.q.out index b952fa6..b767a32 100644 --- a/ql/src/test/results/clientpositive/annotate_stats_part.q.out +++ b/ql/src/test/results/clientpositive/annotate_stats_part.q.out @@ -135,11 +135,11 @@ STAGE PLANS: Processor Tree: TableScan alias: loc_orc - Statistics: Num rows: 2 Data size: 323 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 323 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: string) + expressions: state (type: string), locid (type: int), zip (type: bigint), '__HIVE_DEFAULT_PARTITION__' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2 Data size: 323 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 2 Data size: 323 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: -- basicStatState: PARTIAL colStatState: NONE @@ -181,11 +181,11 @@ STAGE PLANS: Processor Tree: TableScan alias: loc_orc - Statistics: Num rows: 7 Data size: 400 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 7 Data size: 400 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: string) + expressions: state (type: string), locid (type: int), zip (type: bigint), '2001' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 7 Data size: 400 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 7 Data size: 400 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: -- partition level analyze statistics for all partitions @@ -222,11 +222,11 @@ STAGE PLANS: Processor Tree: TableScan alias: loc_orc - Statistics: Num rows: 1 Data size: 323 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 323 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: string) + expressions: state (type: string), locid (type: int), zip (type: bigint), '__HIVE_DEFAULT_PARTITION__' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 323 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 323 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: -- basicStatState: COMPLETE colStatState: NONE @@ -535,14 +535,14 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (locid > 0) (type: boolean) - Statistics: Num rows: 2 Data size: 176 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: locid (type: int), year (type: string) + expressions: locid (type: int), '2001' (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -571,14 +571,14 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (locid > 0) (type: boolean) - Statistics: Num rows: 2 Data size: 176 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: locid (type: int), year (type: string) + expressions: locid (type: int), '2001' (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 176 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 176 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/authorization_role_grant2.q.out b/ql/src/test/results/clientpositive/authorization_role_grant2.q.out index 27d4f33..3db507e 100644 --- a/ql/src/test/results/clientpositive/authorization_role_grant2.q.out +++ b/ql/src/test/results/clientpositive/authorization_role_grant2.q.out @@ -41,6 +41,12 @@ PREHOOK: query: set role src_role_WadMin PREHOOK: type: SHOW_ROLES POSTHOOK: query: set role src_role_WadMin POSTHOOK: type: SHOW_ROLES +PREHOOK: query: show principals src_role_wadmin +PREHOOK: type: SHOW_ROLE_PRINCIPALS +POSTHOOK: query: show principals src_role_wadmin +POSTHOOK: type: SHOW_ROLE_PRINCIPALS +principal_name principal_type grant_option grantor grantor_type grant_time +user2 USER true hive_admin_user USER -1 PREHOOK: query: -- grant role to another user grant src_Role_wadmin to user user3 PREHOOK: type: GRANT_ROLE diff --git a/ql/src/test/results/clientpositive/auto_join11.q.out b/ql/src/test/results/clientpositive/auto_join11.q.out index 3a4658a..8dbad67 100644 --- a/ql/src/test/results/clientpositive/auto_join11.q.out +++ b/ql/src/test/results/clientpositive/auto_join11.q.out @@ -32,12 +32,12 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 100) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 100) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: 0 @@ -53,12 +53,12 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 100) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 100) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -69,11 +69,11 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col3 - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) outputColumnNames: _col0, _col3 - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col0,_col3)) mode: hash diff --git a/ql/src/test/results/clientpositive/auto_join12.q.out b/ql/src/test/results/clientpositive/auto_join12.q.out index cf65a99..037116c 100644 --- a/ql/src/test/results/clientpositive/auto_join12.q.out +++ b/ql/src/test/results/clientpositive/auto_join12.q.out @@ -41,12 +41,12 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key < 80) and key is not null) and (key < 100)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 80) and (key < 100)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: 0 @@ -61,12 +61,12 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key < 100) and key is not null) and (key < 80)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: (((key < 100) and (key < 80)) and key is not null) (type: boolean) + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: 0 @@ -84,12 +84,12 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key < 100) and key is not null) and (key < 80)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 100) and (key < 80)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -103,11 +103,11 @@ STAGE PLANS: 1 _col0 (type: string) 2 _col0 (type: string) outputColumnNames: _col0, _col3 - Statistics: Num rows: 59 Data size: 629 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) outputColumnNames: _col0, _col3 - Statistics: Num rows: 59 Data size: 629 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col0,_col3)) mode: hash diff --git a/ql/src/test/results/clientpositive/auto_join13.q.out b/ql/src/test/results/clientpositive/auto_join13.q.out index cfdaebd..0cb9b4f 100644 --- a/ql/src/test/results/clientpositive/auto_join13.q.out +++ b/ql/src/test/results/clientpositive/auto_join13.q.out @@ -41,12 +41,12 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 100) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 100) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: 0 @@ -80,12 +80,12 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 100) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 100) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -96,10 +96,10 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col2, _col3 - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_col0 + _col2) is not null (type: boolean) - Statistics: Num rows: 46 Data size: 489 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -110,11 +110,11 @@ STAGE PLANS: 0 (_col0 + _col2) (type: double) 1 UDFToDouble(_col0) (type: double) outputColumnNames: _col0, _col3 - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 100 Data size: 1065 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) outputColumnNames: _col0, _col3 - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 100 Data size: 1065 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col0,_col3)) mode: hash diff --git a/ql/src/test/results/clientpositive/auto_join14.q.out b/ql/src/test/results/clientpositive/auto_join14.q.out index addebe4..450a961 100644 --- a/ql/src/test/results/clientpositive/auto_join14.q.out +++ b/ql/src/test/results/clientpositive/auto_join14.q.out @@ -37,8 +37,8 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > 100) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key > 100) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: 0 diff --git a/ql/src/test/results/clientpositive/auto_join16.q.out b/ql/src/test/results/clientpositive/auto_join16.q.out index ac0daa3..f96bae3 100644 --- a/ql/src/test/results/clientpositive/auto_join16.q.out +++ b/ql/src/test/results/clientpositive/auto_join16.q.out @@ -32,12 +32,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((key > 10) and (key > 20)) and key is not null) and value is not null) and (value < 200)) (type: boolean) - Statistics: Num rows: 4 Data size: 42 Basic stats: COMPLETE Column stats: NONE + predicate: ((((key > 10) and value is not null) and (key > 20)) and (value < 200)) (type: boolean) + Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: 0 @@ -53,7 +53,7 @@ STAGE PLANS: alias: tab Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((key > 20) and key is not null) and value is not null) and (value < 200)) (type: boolean) + predicate: ((((key > 20) and value is not null) and key is not null) and (value < 200)) (type: boolean) Statistics: Num rows: 14 Data size: 148 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: diff --git a/ql/src/test/results/clientpositive/auto_join27.q.out b/ql/src/test/results/clientpositive/auto_join27.q.out index 0fc7dd2..c16992f 100644 --- a/ql/src/test/results/clientpositive/auto_join27.q.out +++ b/ql/src/test/results/clientpositive/auto_join27.q.out @@ -37,32 +37,32 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 200) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 200) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: key, value - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: key (type: string), value (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -82,12 +82,12 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 200) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 200) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: 0 @@ -101,7 +101,7 @@ STAGE PLANS: Map Operator Tree: TableScan Union - Statistics: Num rows: 124 Data size: 1316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 249 Data size: 2644 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -111,9 +111,9 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 136 Data size: 1447 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 273 Data size: 2908 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 136 Data size: 1447 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 273 Data size: 2908 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) mode: hash @@ -129,14 +129,14 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 200) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 200) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 124 Data size: 1316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 249 Data size: 2644 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -146,9 +146,9 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - Statistics: Num rows: 136 Data size: 1447 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 273 Data size: 2908 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 136 Data size: 1447 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 273 Data size: 2908 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) mode: hash diff --git a/ql/src/test/results/clientpositive/auto_join29.q.out b/ql/src/test/results/clientpositive/auto_join29.q.out index 26ed0e8..b382a12 100644 --- a/ql/src/test/results/clientpositive/auto_join29.q.out +++ b/ql/src/test/results/clientpositive/auto_join29.q.out @@ -3551,8 +3551,8 @@ STAGE PLANS: alias: src1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key > 10) and key is not null) and (key < 10)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > 10) and (key < 10)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: 0 {value} @@ -3567,8 +3567,8 @@ STAGE PLANS: alias: src2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key < 10) and key is not null) and (key > 10)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 10) and (key > 10)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: 0 {value} @@ -3586,8 +3586,8 @@ STAGE PLANS: alias: src3 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key > 10) and key is not null) and (key < 10)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > 10) and (key < 10)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -3601,25 +3601,25 @@ STAGE PLANS: 1 key (type: string) 2 key (type: string) outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 59 Data size: 629 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 59 Data size: 629 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) sort order: ++++++ - Statistics: Num rows: 59 Data size: 629 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE Local Work: Map Reduce Local Work Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 59 Data size: 629 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 59 Data size: 629 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/auto_join_without_localtask.q.out b/ql/src/test/results/clientpositive/auto_join_without_localtask.q.out index 8762242..318b548 100644 --- a/ql/src/test/results/clientpositive/auto_join_without_localtask.q.out +++ b/ql/src/test/results/clientpositive/auto_join_without_localtask.q.out @@ -613,7 +613,7 @@ STAGE PLANS: TableScan alias: b Filter Operator - predicate: (key is not null and (key > 100)) (type: boolean) + predicate: (key > 100) (type: boolean) HashTable Sink Operator condition expressions: 0 {key} {value} @@ -745,7 +745,7 @@ STAGE PLANS: key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) TableScan alias: c @@ -808,7 +808,7 @@ STAGE PLANS: TableScan alias: b Filter Operator - predicate: (key is not null and (key > 100)) (type: boolean) + predicate: (key > 100) (type: boolean) Map Join Operator condition map: Inner Join 0 to 1 @@ -835,13 +835,13 @@ STAGE PLANS: alias: b Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key > 100)) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key > 100) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE @@ -862,7 +862,7 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: diff --git a/ql/src/test/results/clientpositive/auto_smb_mapjoin_14.q.out b/ql/src/test/results/clientpositive/auto_smb_mapjoin_14.q.out index 011f7bf..2583d9a 100644 --- a/ql/src/test/results/clientpositive/auto_smb_mapjoin_14.q.out +++ b/ql/src/test/results/clientpositive/auto_smb_mapjoin_14.q.out @@ -506,12 +506,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -853,12 +853,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 8) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 8) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -1072,12 +1072,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -1167,12 +1167,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 diff --git a/ql/src/test/results/clientpositive/auto_sortmerge_join_10.q.out b/ql/src/test/results/clientpositive/auto_sortmerge_join_10.q.out index 488fc50..7606dd2 100644 --- a/ql/src/test/results/clientpositive/auto_sortmerge_join_10.q.out +++ b/ql/src/test/results/clientpositive/auto_sortmerge_join_10.q.out @@ -83,12 +83,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: 0 @@ -104,18 +104,18 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 4 Data size: 28 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 42 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 28 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 42 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -125,9 +125,9 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 46 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 46 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -143,18 +143,18 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 4 Data size: 28 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 42 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 28 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 42 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -164,9 +164,9 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 46 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 46 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -272,24 +272,24 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: key - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() bucketGroup: true keys: key (type: int) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -321,12 +321,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: 0 @@ -348,9 +348,9 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - Statistics: Num rows: 2 Data size: 15 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 2 Data size: 15 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash diff --git a/ql/src/test/results/clientpositive/auto_sortmerge_join_9.q.out b/ql/src/test/results/clientpositive/auto_sortmerge_join_9.q.out index 95b264f..1a7fdf9 100644 --- a/ql/src/test/results/clientpositive/auto_sortmerge_join_9.q.out +++ b/ql/src/test/results/clientpositive/auto_sortmerge_join_9.q.out @@ -699,12 +699,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -1046,12 +1046,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 8) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 8) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -1264,12 +1264,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -1444,12 +1444,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -3012,7 +3012,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) + predicate: (key < 6) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -3030,7 +3030,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) + predicate: (key < 6) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -3079,7 +3079,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) + predicate: (key < 6) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -3097,7 +3097,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) + predicate: (key < 6) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -3142,12 +3142,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -3779,7 +3779,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: ((key < 8) and key is not null) (type: boolean) + predicate: (key < 8) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -3797,7 +3797,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: ((key < 8) and key is not null) (type: boolean) + predicate: (key < 8) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -3846,7 +3846,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: ((key < 8) and key is not null) (type: boolean) + predicate: (key < 8) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -3864,7 +3864,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: ((key < 8) and key is not null) (type: boolean) + predicate: (key < 8) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -3909,12 +3909,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 8) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 8) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -4025,7 +4025,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) + predicate: (key < 6) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -4074,7 +4074,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) + predicate: (key < 6) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -4134,12 +4134,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -4233,7 +4233,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) + predicate: (key < 6) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -4312,7 +4312,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) + predicate: (key < 6) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -4467,7 +4467,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) + predicate: (key < 6) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -4484,7 +4484,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) + predicate: (key < 6) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -4504,7 +4504,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) + predicate: (key < 6) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -4559,7 +4559,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) + predicate: (key < 6) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -4576,7 +4576,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) + predicate: (key < 6) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -4596,7 +4596,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) + predicate: (key < 6) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -4651,7 +4651,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) + predicate: (key < 6) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -4668,7 +4668,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) + predicate: (key < 6) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -4688,7 +4688,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) + predicate: (key < 6) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -4736,12 +4736,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 diff --git a/ql/src/test/results/clientpositive/avro_date.q.out b/ql/src/test/results/clientpositive/avro_date.q.out new file mode 100644 index 0000000..276a53c --- /dev/null +++ b/ql/src/test/results/clientpositive/avro_date.q.out @@ -0,0 +1,126 @@ +PREHOOK: query: DROP TABLE avro_date_staging +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE avro_date_staging +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE avro_date +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE avro_date +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE avro_date_casts +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE avro_date_casts +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE avro_date_staging (d date, m1 map, l1 array) + ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' + COLLECTION ITEMS TERMINATED BY ',' MAP KEYS TERMINATED BY ':' + STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@avro_date_staging +POSTHOOK: query: CREATE TABLE avro_date_staging (d date, m1 map, l1 array) + ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' + COLLECTION ITEMS TERMINATED BY ',' MAP KEYS TERMINATED BY ':' + STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@avro_date_staging +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/avro_date.txt' OVERWRITE INTO TABLE avro_date_staging +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@avro_date_staging +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/avro_date.txt' OVERWRITE INTO TABLE avro_date_staging +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@avro_date_staging +PREHOOK: query: CREATE TABLE avro_date (d date, m1 map, l1 array) + PARTITIONED BY (p1 int, p2 date) + ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' + COLLECTION ITEMS TERMINATED BY ',' MAP KEYS TERMINATED BY ':' + STORED AS AVRO +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@avro_date +POSTHOOK: query: CREATE TABLE avro_date (d date, m1 map, l1 array) + PARTITIONED BY (p1 int, p2 date) + ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' + COLLECTION ITEMS TERMINATED BY ',' MAP KEYS TERMINATED BY ':' + STORED AS AVRO +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@avro_date +PREHOOK: query: INSERT OVERWRITE TABLE avro_date PARTITION(p1=2, p2='2014-09-26') SELECT * FROM avro_date_staging +PREHOOK: type: QUERY +PREHOOK: Input: default@avro_date_staging +PREHOOK: Output: default@avro_date@p1=2/p2=2014-09-26 +POSTHOOK: query: INSERT OVERWRITE TABLE avro_date PARTITION(p1=2, p2='2014-09-26') SELECT * FROM avro_date_staging +POSTHOOK: type: QUERY +POSTHOOK: Input: default@avro_date_staging +POSTHOOK: Output: default@avro_date@p1=2/p2=2014-09-26 +POSTHOOK: Lineage: avro_date PARTITION(p1=2,p2=2014-09-26).d SIMPLE [(avro_date_staging)avro_date_staging.FieldSchema(name:d, type:date, comment:null), ] +POSTHOOK: Lineage: avro_date PARTITION(p1=2,p2=2014-09-26).l1 SIMPLE [(avro_date_staging)avro_date_staging.FieldSchema(name:l1, type:array, comment:null), ] +POSTHOOK: Lineage: avro_date PARTITION(p1=2,p2=2014-09-26).m1 SIMPLE [(avro_date_staging)avro_date_staging.FieldSchema(name:m1, type:map, comment:null), ] +PREHOOK: query: SELECT * FROM avro_date +PREHOOK: type: QUERY +PREHOOK: Input: default@avro_date +PREHOOK: Input: default@avro_date@p1=2/p2=2014-09-26 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM avro_date +POSTHOOK: type: QUERY +POSTHOOK: Input: default@avro_date +POSTHOOK: Input: default@avro_date@p1=2/p2=2014-09-26 +#### A masked pattern was here #### +2012-02-21 {"foo":"1980-12-16","bar":"1998-05-07"} ["2011-09-04","2011-09-05"] 2 2014-09-26 +2014-02-11 {"baz":"1981-12-16"} ["2011-09-05"] 2 2014-09-26 +1947-02-11 {"baz":"1921-12-16"} ["2011-09-05"] 2 2014-09-26 +8200-02-11 {"baz":"6981-12-16"} ["1039-09-05"] 2 2014-09-26 +PREHOOK: query: SELECT d, COUNT(d) FROM avro_date GROUP BY d +PREHOOK: type: QUERY +PREHOOK: Input: default@avro_date +PREHOOK: Input: default@avro_date@p1=2/p2=2014-09-26 +#### A masked pattern was here #### +POSTHOOK: query: SELECT d, COUNT(d) FROM avro_date GROUP BY d +POSTHOOK: type: QUERY +POSTHOOK: Input: default@avro_date +POSTHOOK: Input: default@avro_date@p1=2/p2=2014-09-26 +#### A masked pattern was here #### +1947-02-11 1 +2012-02-21 1 +2014-02-11 1 +8200-02-11 1 +PREHOOK: query: SELECT * FROM avro_date WHERE d!='1947-02-11' +PREHOOK: type: QUERY +PREHOOK: Input: default@avro_date +PREHOOK: Input: default@avro_date@p1=2/p2=2014-09-26 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM avro_date WHERE d!='1947-02-11' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@avro_date +POSTHOOK: Input: default@avro_date@p1=2/p2=2014-09-26 +#### A masked pattern was here #### +2012-02-21 {"foo":"1980-12-16","bar":"1998-05-07"} ["2011-09-04","2011-09-05"] 2 2014-09-26 +2014-02-11 {"baz":"1981-12-16"} ["2011-09-05"] 2 2014-09-26 +8200-02-11 {"baz":"6981-12-16"} ["1039-09-05"] 2 2014-09-26 +PREHOOK: query: SELECT * FROM avro_date WHERE d<'2014-12-21' +PREHOOK: type: QUERY +PREHOOK: Input: default@avro_date +PREHOOK: Input: default@avro_date@p1=2/p2=2014-09-26 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM avro_date WHERE d<'2014-12-21' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@avro_date +POSTHOOK: Input: default@avro_date@p1=2/p2=2014-09-26 +#### A masked pattern was here #### +2012-02-21 {"foo":"1980-12-16","bar":"1998-05-07"} ["2011-09-04","2011-09-05"] 2 2014-09-26 +2014-02-11 {"baz":"1981-12-16"} ["2011-09-05"] 2 2014-09-26 +1947-02-11 {"baz":"1921-12-16"} ["2011-09-05"] 2 2014-09-26 +PREHOOK: query: SELECT * FROM avro_date WHERE d>'8000-12-01' +PREHOOK: type: QUERY +PREHOOK: Input: default@avro_date +PREHOOK: Input: default@avro_date@p1=2/p2=2014-09-26 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM avro_date WHERE d>'8000-12-01' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@avro_date +POSTHOOK: Input: default@avro_date@p1=2/p2=2014-09-26 +#### A masked pattern was here #### +8200-02-11 {"baz":"6981-12-16"} ["1039-09-05"] 2 2014-09-26 diff --git a/ql/src/test/results/clientpositive/avro_deserialize_map_null.q.out b/ql/src/test/results/clientpositive/avro_deserialize_map_null.q.out new file mode 100644 index 0000000..de79d62 --- /dev/null +++ b/ql/src/test/results/clientpositive/avro_deserialize_map_null.q.out @@ -0,0 +1,55 @@ +PREHOOK: query: -- These test attempts to deserialize an Avro file that contains map null values, and the file schema +-- vs record schema have the null values in different positions +-- i.e. +-- fileSchema = [{ "type" : "map", "values" : ["string","null"]}, "null"] +-- recordSchema = ["null", { "type" : "map", "values" : ["string","null"]}] + + +DROP TABLE IF EXISTS avro_table +PREHOOK: type: DROPTABLE +POSTHOOK: query: -- These test attempts to deserialize an Avro file that contains map null values, and the file schema +-- vs record schema have the null values in different positions +-- i.e. +-- fileSchema = [{ "type" : "map", "values" : ["string","null"]}, "null"] +-- recordSchema = ["null", { "type" : "map", "values" : ["string","null"]}] + + +DROP TABLE IF EXISTS avro_table +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE avro_table (avreau_col_1 map) STORED AS AVRO +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@avro_table +POSTHOOK: query: CREATE TABLE avro_table (avreau_col_1 map) STORED AS AVRO +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@avro_table +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/map_null_val.avro' OVERWRITE INTO TABLE avro_table +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@avro_table +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/map_null_val.avro' OVERWRITE INTO TABLE avro_table +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@avro_table +PREHOOK: query: SELECT * FROM avro_table +PREHOOK: type: QUERY +PREHOOK: Input: default@avro_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM avro_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@avro_table +#### A masked pattern was here #### +{"key4":null,"key3":"val3"} +{"key4":null,"key3":"val3"} +{"key2":"val2","key1":null} +{"key4":null,"key3":"val3"} +{"key4":null,"key3":"val3"} +PREHOOK: query: DROP TABLE avro_table +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@avro_table +PREHOOK: Output: default@avro_table +POSTHOOK: query: DROP TABLE avro_table +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@avro_table +POSTHOOK: Output: default@avro_table diff --git a/ql/src/test/results/clientpositive/avro_schema_error_message.q.out b/ql/src/test/results/clientpositive/avro_schema_error_message.q.out index 967a847..e69de29 100644 --- a/ql/src/test/results/clientpositive/avro_schema_error_message.q.out +++ b/ql/src/test/results/clientpositive/avro_schema_error_message.q.out @@ -1,35 +0,0 @@ -PREHOOK: query: -- verify we get the sentinel schema if we don't provide one - -CREATE TABLE avro_with_no_schema -ROW FORMAT -SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' -STORED AS -INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' -OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@avro_with_no_schema -POSTHOOK: query: -- verify we get the sentinel schema if we don't provide one - -CREATE TABLE avro_with_no_schema -ROW FORMAT -SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' -STORED AS -INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' -OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@avro_with_no_schema -PREHOOK: query: DESCRIBE avro_with_no_schema -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@avro_with_no_schema -POSTHOOK: query: DESCRIBE avro_with_no_schema -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@avro_with_no_schema -error_error_error_error_error_error_error string from deserializer -cannot_determine_schema string from deserializer -check string from deserializer -schema string from deserializer -url string from deserializer -and string from deserializer -literal string from deserializer diff --git a/ql/src/test/results/clientpositive/bucket3.q.out b/ql/src/test/results/clientpositive/bucket3.q.out index 7b65e52..0d6516c 100644 --- a/ql/src/test/results/clientpositive/bucket3.q.out +++ b/ql/src/test/results/clientpositive/bucket3.q.out @@ -227,7 +227,7 @@ STAGE PLANS: predicate: (((hash(key) & 2147483647) % 2) = 0) (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: int), value (type: string), ds (type: string) + expressions: key (type: int), value (type: string), '1' (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/cast_qualified_types.q.out b/ql/src/test/results/clientpositive/cast_qualified_types.q.out new file mode 100644 index 0000000..1924c5d --- /dev/null +++ b/ql/src/test/results/clientpositive/cast_qualified_types.q.out @@ -0,0 +1,21 @@ +PREHOOK: query: select + cast(key as decimal(10,2)) as c1, + cast(key as char(10)) as c2, + cast(key as varchar(10)) as c3 +from src +order by c1, c2, c3 +limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select + cast(key as decimal(10,2)) as c1, + cast(key as char(10)) as c2, + cast(key as varchar(10)) as c3 +from src +order by c1, c2, c3 +limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 0 0 diff --git a/ql/src/test/results/clientpositive/cbo_correctness.q.out b/ql/src/test/results/clientpositive/cbo_correctness.q.out index 3335d4d..d98cb5b 100644 --- a/ql/src/test/results/clientpositive/cbo_correctness.q.out +++ b/ql/src/test/results/clientpositive/cbo_correctness.q.out @@ -16719,7 +16719,7 @@ POSTHOOK: type: DROPVIEW POSTHOOK: Input: default@v4 POSTHOOK: Output: default@v4 PREHOOK: query: -- 11. Union All -select * from t1 union all select * from t2 order by key +select * from t1 union all select * from t2 order by key, c_boolean, value, dt PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t1@dt=2014 @@ -16727,7 +16727,7 @@ PREHOOK: Input: default@t2 PREHOOK: Input: default@t2@dt=2014 #### A masked pattern was here #### POSTHOOK: query: -- 11. Union All -select * from t1 union all select * from t2 order by key +select * from t1 union all select * from t2 order by key, c_boolean, value, dt POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t1@dt=2014 @@ -16738,13 +16738,13 @@ POSTHOOK: Input: default@t2@dt=2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 -1 1 1 1.0 true 2014 1 1 1 1.0 false 2014 -1 1 1 1.0 true 2014 1 1 1 1.0 false 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 +1 1 1 1.0 true 2014 +1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 2 2 2 2.0 true 2014 @@ -18142,7 +18142,7 @@ from src_cbo where src_cbo.key not in ( select key from src_cbo s1 where s1.key > '2' - ) + ) order by key PREHOOK: type: QUERY PREHOOK: Input: default@src_cbo #### A masked pattern was here #### @@ -18153,7 +18153,7 @@ from src_cbo where src_cbo.key not in ( select key from src_cbo s1 where s1.key > '2' - ) + ) order by key POSTHOOK: type: QUERY POSTHOOK: Input: default@src_cbo #### A masked pattern was here #### @@ -18946,17 +18946,162 @@ POSTHOOK: Input: default@src_cbo 96 1 97 2 98 2 -PREHOOK: query: -- 17. get stats with empty partition list +PREHOOK: query: -- 20. Test get stats with empty partition list select t1.value from t1 join t2 on t1.key = t2.key where t1.dt = '10' and t1.c_boolean = true PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 PREHOOK: Input: default@t2@dt=2014 #### A masked pattern was here #### -POSTHOOK: query: -- 17. get stats with empty partition list +POSTHOOK: query: -- 20. Test get stats with empty partition list select t1.value from t1 join t2 on t1.key = t2.key where t1.dt = '10' and t1.c_boolean = true POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t2 POSTHOOK: Input: default@t2@dt=2014 #### A masked pattern was here #### +PREHOOK: query: -- 21. Test groupby is empty and there is no other cols in aggr +select unionsrc.key FROM (select 'tst1' as key, count(1) as value from src) unionsrc +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: -- 21. Test groupby is empty and there is no other cols in aggr +select unionsrc.key FROM (select 'tst1' as key, count(1) as value from src) unionsrc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +tst1 +PREHOOK: query: select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src) unionsrc +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src) unionsrc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +tst1 500 +PREHOOK: query: select unionsrc.key FROM (select 'max' as key, max(c_int) as value from t3 s1 + UNION ALL + select 'min' as key, min(c_int) as value from t3 s2 + UNION ALL + select 'avg' as key, avg(c_int) as value from t3 s3) unionsrc order by unionsrc.key +PREHOOK: type: QUERY +PREHOOK: Input: default@t3 +#### A masked pattern was here #### +POSTHOOK: query: select unionsrc.key FROM (select 'max' as key, max(c_int) as value from t3 s1 + UNION ALL + select 'min' as key, min(c_int) as value from t3 s2 + UNION ALL + select 'avg' as key, avg(c_int) as value from t3 s3) unionsrc order by unionsrc.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t3 +#### A masked pattern was here #### +avg +max +min +PREHOOK: query: select unionsrc.key, unionsrc.value FROM (select 'max' as key, max(c_int) as value from t3 s1 + UNION ALL + select 'min' as key, min(c_int) as value from t3 s2 + UNION ALL + select 'avg' as key, avg(c_int) as value from t3 s3) unionsrc order by unionsrc.key +PREHOOK: type: QUERY +PREHOOK: Input: default@t3 +#### A masked pattern was here #### +POSTHOOK: query: select unionsrc.key, unionsrc.value FROM (select 'max' as key, max(c_int) as value from t3 s1 + UNION ALL + select 'min' as key, min(c_int) as value from t3 s2 + UNION ALL + select 'avg' as key, avg(c_int) as value from t3 s3) unionsrc order by unionsrc.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t3 +#### A masked pattern was here #### +avg 1.5 +max 3.0 +min 1.0 +PREHOOK: query: select unionsrc.key, count(1) FROM (select 'max' as key, max(c_int) as value from t3 s1 + UNION ALL + select 'min' as key, min(c_int) as value from t3 s2 + UNION ALL + select 'avg' as key, avg(c_int) as value from t3 s3) unionsrc group by unionsrc.key order by unionsrc.key +PREHOOK: type: QUERY +PREHOOK: Input: default@t3 +#### A masked pattern was here #### +POSTHOOK: query: select unionsrc.key, count(1) FROM (select 'max' as key, max(c_int) as value from t3 s1 + UNION ALL + select 'min' as key, min(c_int) as value from t3 s2 + UNION ALL + select 'avg' as key, avg(c_int) as value from t3 s3) unionsrc group by unionsrc.key order by unionsrc.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t3 +#### A masked pattern was here #### +avg 1 +max 1 +min 1 +PREHOOK: query: -- Windowing +select *, rank() over(partition by key order by value) as rr from src1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: -- Windowing +select *, rank() over(partition by key order by value) as rr from src1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### + 1 + 1 + 1 + 1 + val_165 5 + val_193 6 + val_265 7 + val_27 8 + val_409 9 + val_484 10 +128 1 +146 val_146 1 +150 val_150 1 +213 val_213 1 +224 1 +238 val_238 1 +255 val_255 1 +273 val_273 1 +278 val_278 1 +311 val_311 1 +369 1 +401 val_401 1 +406 val_406 1 +66 val_66 1 +98 val_98 1 +PREHOOK: query: select *, rank() over(partition by key order by value) from src1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: select *, rank() over(partition by key order by value) from src1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### + 1 + 1 + 1 + 1 + val_165 5 + val_193 6 + val_265 7 + val_27 8 + val_409 9 + val_484 10 +128 1 +146 val_146 1 +150 val_150 1 +213 val_213 1 +224 1 +238 val_238 1 +255 val_255 1 +273 val_273 1 +278 val_278 1 +311 val_311 1 +369 1 +401 val_401 1 +406 val_406 1 +66 val_66 1 +98 val_98 1 diff --git a/ql/src/test/results/clientpositive/cluster.q.out b/ql/src/test/results/clientpositive/cluster.q.out index 5808c74..f5573c9 100644 --- a/ql/src/test/results/clientpositive/cluster.q.out +++ b/ql/src/test/results/clientpositive/cluster.q.out @@ -473,22 +473,22 @@ STAGE PLANS: alias: y Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key = 20)) (type: boolean) - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 20) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: '20' (type: string) sort order: + - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE TableScan alias: x Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key = 20)) (type: boolean) - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 20) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: '20' (type: string) sort order: + - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Reduce Operator Tree: Join Operator @@ -498,11 +498,11 @@ STAGE PLANS: 0 {VALUE._col0} 1 outputColumnNames: _col1 - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -518,15 +518,15 @@ STAGE PLANS: key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator expressions: '20' (type: string), KEY.reducesinkkey0 (type: string), '20' (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -566,23 +566,23 @@ STAGE PLANS: alias: y Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key = 20)) (type: boolean) - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 20) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: '20' (type: string) sort order: + - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) TableScan alias: x Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key = 20)) (type: boolean) - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 20) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: '20' (type: string) sort order: + - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Reduce Operator Tree: Join Operator @@ -592,11 +592,11 @@ STAGE PLANS: 0 {VALUE._col0} 1 {VALUE._col0} outputColumnNames: _col1, _col6 - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col6 (type: string) outputColumnNames: _col1, _col3 - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -612,16 +612,16 @@ STAGE PLANS: key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE value expressions: _col3 (type: string) Reduce Operator Tree: Select Operator expressions: '20' (type: string), KEY.reducesinkkey0 (type: string), '20' (type: string), VALUE._col2 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -661,23 +661,23 @@ STAGE PLANS: alias: y Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key = 20)) (type: boolean) - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 20) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: '20' (type: string) sort order: + - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) TableScan alias: x Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key = 20)) (type: boolean) - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 20) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: '20' (type: string) sort order: + - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Reduce Operator Tree: Join Operator @@ -687,11 +687,11 @@ STAGE PLANS: 0 {VALUE._col0} 1 {VALUE._col0} outputColumnNames: _col1, _col6 - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col6 (type: string) outputColumnNames: _col1, _col3 - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -706,16 +706,16 @@ STAGE PLANS: Reduce Output Operator key expressions: '20' (type: string) sort order: + - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col3 (type: string) Reduce Operator Tree: Select Operator expressions: '20' (type: string), VALUE._col0 (type: string), '20' (type: string), VALUE._col2 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -755,22 +755,22 @@ STAGE PLANS: alias: y Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key = 20)) (type: boolean) - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 20) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: '20' (type: string) sort order: + - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE TableScan alias: x Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key = 20)) (type: boolean) - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 20) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: '20' (type: string) sort order: + - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Reduce Operator Tree: Join Operator @@ -780,11 +780,11 @@ STAGE PLANS: 0 {VALUE._col0} 1 outputColumnNames: _col1 - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), '20' (type: string) outputColumnNames: _col1, _col2 - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -799,16 +799,16 @@ STAGE PLANS: Reduce Output Operator key expressions: '20' (type: string) sort order: + - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col2 (type: string) Reduce Operator Tree: Select Operator expressions: '20' (type: string), VALUE._col0 (type: string), VALUE._col1 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/columnstats_partlvl.q.out b/ql/src/test/results/clientpositive/columnstats_partlvl.q.out index 626541c..0131517 100644 --- a/ql/src/test/results/clientpositive/columnstats_partlvl.q.out +++ b/ql/src/test/results/clientpositive/columnstats_partlvl.q.out @@ -47,7 +47,7 @@ STAGE PLANS: TableScan alias: employee_part Select Operator - expressions: employeesalary (type: double), employeeid (type: int) + expressions: 2000.0 (type: double), employeeid (type: int) outputColumnNames: employeesalary, employeeid Group By Operator aggregations: compute_stats(employeeid, 16) @@ -115,7 +115,7 @@ STAGE PLANS: alias: employee_part GatherStats: false Select Operator - expressions: employeesalary (type: double), employeeid (type: int) + expressions: 2000.0 (type: double), employeeid (type: int) outputColumnNames: employeesalary, employeeid Group By Operator aggregations: compute_stats(employeeid, 16) @@ -247,7 +247,7 @@ STAGE PLANS: TableScan alias: employee_part Select Operator - expressions: employeesalary (type: double), employeeid (type: int) + expressions: 4000.0 (type: double), employeeid (type: int) outputColumnNames: employeesalary, employeeid Group By Operator aggregations: compute_stats(employeeid, 16) @@ -315,7 +315,7 @@ STAGE PLANS: alias: employee_part GatherStats: false Select Operator - expressions: employeesalary (type: double), employeeid (type: int) + expressions: 4000.0 (type: double), employeeid (type: int) outputColumnNames: employeesalary, employeeid Group By Operator aggregations: compute_stats(employeeid, 16) @@ -447,7 +447,7 @@ STAGE PLANS: TableScan alias: employee_part Select Operator - expressions: employeesalary (type: double), employeeid (type: int), employeename (type: string) + expressions: 2000.0 (type: double), employeeid (type: int), employeename (type: string) outputColumnNames: employeesalary, employeeid, employeename Group By Operator aggregations: compute_stats(employeeid, 16), compute_stats(employeename, 16) diff --git a/ql/src/test/results/clientpositive/columnstats_partlvl_dp.q.out b/ql/src/test/results/clientpositive/columnstats_partlvl_dp.q.out index 1c95486..073b387 100644 --- a/ql/src/test/results/clientpositive/columnstats_partlvl_dp.q.out +++ b/ql/src/test/results/clientpositive/columnstats_partlvl_dp.q.out @@ -85,7 +85,7 @@ STAGE PLANS: TableScan alias: employee_part Select Operator - expressions: employeesalary (type: double), country (type: string), employeename (type: string), employeeid (type: int) + expressions: 4000.0 (type: double), country (type: string), employeename (type: string), employeeid (type: int) outputColumnNames: employeesalary, country, employeename, employeeid Group By Operator aggregations: compute_stats(employeename, 16), compute_stats(employeeid, 16) @@ -158,7 +158,7 @@ STAGE PLANS: TableScan alias: employee_part Select Operator - expressions: employeesalary (type: double), country (type: string), employeeid (type: int) + expressions: 2000.0 (type: double), country (type: string), employeeid (type: int) outputColumnNames: employeesalary, country, employeeid Group By Operator aggregations: compute_stats(employeeid, 16) diff --git a/ql/src/test/results/clientpositive/complex_alias.q.out b/ql/src/test/results/clientpositive/complex_alias.q.out new file mode 100644 index 0000000..d8264bd --- /dev/null +++ b/ql/src/test/results/clientpositive/complex_alias.q.out @@ -0,0 +1,269 @@ +PREHOOK: query: CREATE TABLE agg1 (col0 INT, col1 STRING, col2 DOUBLE) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@agg1 +POSTHOOK: query: CREATE TABLE agg1 (col0 INT, col1 STRING, col2 DOUBLE) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@agg1 +PREHOOK: query: INSERT INTO TABLE agg1 select key,value,key from src tablesample (1 rows) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@agg1 +POSTHOOK: query: INSERT INTO TABLE agg1 select key,value,key from src tablesample (1 rows) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@agg1 +POSTHOOK: Lineage: agg1.col0 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: agg1.col1 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: agg1.col2 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +Warning: Shuffle Join JOIN[19][tables = [single_use_subq12, single_use_subq11]] in Stage 'Stage-2:MAPRED' is a cross product +PREHOOK: query: EXPLAIN +SELECT single_use_subq11.a1 AS a1, + single_use_subq11.a2 AS a2 +FROM (SELECT Sum(agg1.col2) AS a1 + FROM agg1 + GROUP BY agg1.col0) single_use_subq12 + JOIN (SELECT alias.a2 AS a0, + alias.a1 AS a1, + alias.a1 AS a2 + FROM (SELECT agg1.col1 AS a0, + '42' AS a1, + agg1.col0 AS a2 + FROM agg1 + UNION ALL + SELECT agg1.col1 AS a0, + '41' AS a1, + agg1.col0 AS a2 + FROM agg1) alias + GROUP BY alias.a2, + alias.a1) single_use_subq11 + ON ( single_use_subq11.a0 = single_use_subq11.a0 ) +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT single_use_subq11.a1 AS a1, + single_use_subq11.a2 AS a2 +FROM (SELECT Sum(agg1.col2) AS a1 + FROM agg1 + GROUP BY agg1.col0) single_use_subq12 + JOIN (SELECT alias.a2 AS a0, + alias.a1 AS a1, + alias.a1 AS a2 + FROM (SELECT agg1.col1 AS a0, + '42' AS a1, + agg1.col0 AS a2 + FROM agg1 + UNION ALL + SELECT agg1.col1 AS a0, + '41' AS a1, + agg1.col0 AS a2 + FROM agg1) alias + GROUP BY alias.a2, + alias.a1) single_use_subq11 + ON ( single_use_subq11.a0 = single_use_subq11.a0 ) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1, Stage-4 + Stage-4 is a root stage + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: agg1 + Statistics: Num rows: 1 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (col0 = col0) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: '42' (type: string), col0 (type: int) + outputColumnNames: _col1, _col2 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Union + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col2 (type: int), _col1 (type: string) + outputColumnNames: _col2, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Group By Operator + keys: _col2 (type: int), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + TableScan + alias: agg1 + Statistics: Num rows: 1 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (col0 = col0) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: '41' (type: string), col0 (type: int) + outputColumnNames: _col1, _col2 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Union + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col2 (type: int), _col1 (type: string) + outputColumnNames: _col2, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Group By Operator + keys: _col2 (type: int), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col1 (type: string), _col1 (type: string) + outputColumnNames: _col1, _col2 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + sort order: + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: _col1 (type: string), _col2 (type: string) + TableScan + Reduce Output Operator + sort order: + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {VALUE._col1} {VALUE._col2} + outputColumnNames: _col2, _col3 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col2 (type: string), _col3 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + alias: agg1 + Statistics: Num rows: 1 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col0 (type: int), col2 (type: double) + outputColumnNames: col0, col2 + Statistics: Num rows: 1 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(col2) + keys: col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 17 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: double) + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +Warning: Shuffle Join JOIN[19][tables = [single_use_subq12, single_use_subq11]] in Stage 'Stage-2:MAPRED' is a cross product +PREHOOK: query: SELECT single_use_subq11.a1 AS a1, + single_use_subq11.a2 AS a2 +FROM (SELECT Sum(agg1.col2) AS a1 + FROM agg1 + GROUP BY agg1.col0) single_use_subq12 + JOIN (SELECT alias.a2 AS a0, + alias.a1 AS a1, + alias.a1 AS a2 + FROM (SELECT agg1.col1 AS a0, + '42' AS a1, + agg1.col0 AS a2 + FROM agg1 + UNION ALL + SELECT agg1.col1 AS a0, + '41' AS a1, + agg1.col0 AS a2 + FROM agg1) alias + GROUP BY alias.a2, + alias.a1) single_use_subq11 + ON ( single_use_subq11.a0 = single_use_subq11.a0 ) +PREHOOK: type: QUERY +PREHOOK: Input: default@agg1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT single_use_subq11.a1 AS a1, + single_use_subq11.a2 AS a2 +FROM (SELECT Sum(agg1.col2) AS a1 + FROM agg1 + GROUP BY agg1.col0) single_use_subq12 + JOIN (SELECT alias.a2 AS a0, + alias.a1 AS a1, + alias.a1 AS a2 + FROM (SELECT agg1.col1 AS a0, + '42' AS a1, + agg1.col0 AS a2 + FROM agg1 + UNION ALL + SELECT agg1.col1 AS a0, + '41' AS a1, + agg1.col0 AS a2 + FROM agg1) alias + GROUP BY alias.a2, + alias.a1) single_use_subq11 + ON ( single_use_subq11.a0 = single_use_subq11.a0 ) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@agg1 +#### A masked pattern was here #### +42 42 +41 41 diff --git a/ql/src/test/results/clientpositive/constprog2.q.out b/ql/src/test/results/clientpositive/constprog2.q.out index 148d95b..71a44f3 100644 --- a/ql/src/test/results/clientpositive/constprog2.q.out +++ b/ql/src/test/results/clientpositive/constprog2.q.out @@ -18,23 +18,23 @@ STAGE PLANS: alias: src2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key = 86) and key is not null) (type: boolean) - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 86) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: '86' (type: string) sort order: + - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) TableScan alias: src1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key = 86) and key is not null) (type: boolean) - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 86) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: '86' (type: string) sort order: + - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -43,14 +43,14 @@ STAGE PLANS: 0 1 {VALUE._col0} outputColumnNames: _col6 - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: '86' (type: string), 87.0 (type: double), _col6 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -73,3 +73,78 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### 86 87.0 val_86 +PREHOOK: query: EXPLAIN +SELECT src1.key, src1.key + 1, src2.value + FROM src src1 join src src2 ON src1.key = src2.key AND cast(src1.key as double) = 86 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT src1.key, src1.key + 1, src2.value + FROM src src1 join src src2 ON src1.key = src2.key AND cast(src1.key as double) = 86 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) = 86) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: '86' (type: string) + sort order: + + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + TableScan + alias: src1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) = 86) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: '86' (type: string) + sort order: + + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {VALUE._col0} + outputColumnNames: _col6 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '86' (type: string), 87.0 (type: double), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT src1.key, src1.key + 1, src2.value + FROM src src1 join src src2 ON src1.key = src2.key AND cast(src1.key as double) = 86 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT src1.key, src1.key + 1, src2.value + FROM src src1 join src src2 ON src1.key = src2.key AND cast(src1.key as double) = 86 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +86 87.0 val_86 diff --git a/ql/src/test/results/clientpositive/constprog_dp.q.out b/ql/src/test/results/clientpositive/constprog_dp.q.out index 98af017..8cf301d 100644 --- a/ql/src/test/results/clientpositive/constprog_dp.q.out +++ b/ql/src/test/results/clientpositive/constprog_dp.q.out @@ -32,7 +32,7 @@ STAGE PLANS: alias: srcpart Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string), ds (type: string) + expressions: key (type: string), value (type: string), '2008-04-08' (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/constprog_type.q.out b/ql/src/test/results/clientpositive/constprog_type.q.out index 1764184..d145d37 100644 --- a/ql/src/test/results/clientpositive/constprog_type.q.out +++ b/ql/src/test/results/clientpositive/constprog_type.q.out @@ -122,3 +122,69 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@dest1 #### A masked pattern was here #### 2013-11-17 2011-04-29 20:46:56.4485 +PREHOOK: query: SELECT key, value FROM src WHERE key = cast(86 as double) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT key, value FROM src WHERE key = cast(86 as double) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +86 val_86 +PREHOOK: query: CREATE TABLE primitives1 ( + id INT , + bool_col BOOLEAN , + tinyint_col TINYINT , + smallint_col SMALLINT , + int_col INT , + bigint_col BIGINT , + float_col FLOAT , + double_col DOUBLE , + date_string_col STRING , + string_col STRING , + timestamp_col TIMESTAMP ) +ROW FORMAT DELIMITED + FIELDS TERMINATED BY ',' + ESCAPED BY '\\' +STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@primitives1 +POSTHOOK: query: CREATE TABLE primitives1 ( + id INT , + bool_col BOOLEAN , + tinyint_col TINYINT , + smallint_col SMALLINT , + int_col INT , + bigint_col BIGINT , + float_col FLOAT , + double_col DOUBLE , + date_string_col STRING , + string_col STRING , + timestamp_col TIMESTAMP ) +ROW FORMAT DELIMITED + FIELDS TERMINATED BY ',' + ESCAPED BY '\\' +STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@primitives1 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/types/primitives/090101.txt' +OVERWRITE INTO TABLE primitives1 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@primitives1 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/types/primitives/090101.txt' +OVERWRITE INTO TABLE primitives1 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@primitives1 +PREHOOK: query: select id,bool_col,tinyint_col,smallint_col,int_col,bigint_col,float_col,double_col from primitives1 where id = cast (0 as float) and bool_col = cast('true' as boolean) and tinyint_col = cast(0 as double) and smallint_col = cast(0 as bigint) and int_col = cast (0 as double) and bigint_col = cast(0 as tinyint) and float_col = cast(0.0 as string) and double_col = cast (0.0 as float) +PREHOOK: type: QUERY +PREHOOK: Input: default@primitives1 +#### A masked pattern was here #### +POSTHOOK: query: select id,bool_col,tinyint_col,smallint_col,int_col,bigint_col,float_col,double_col from primitives1 where id = cast (0 as float) and bool_col = cast('true' as boolean) and tinyint_col = cast(0 as double) and smallint_col = cast(0 as bigint) and int_col = cast (0 as double) and bigint_col = cast(0 as tinyint) and float_col = cast(0.0 as string) and double_col = cast (0.0 as float) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@primitives1 +#### A masked pattern was here #### +0 true 0 0 0 0 0.0 0.0 diff --git a/ql/src/test/results/clientpositive/correlationoptimizer10.q.out b/ql/src/test/results/clientpositive/correlationoptimizer10.q.out index 27c573b..6e7904a 100644 --- a/ql/src/test/results/clientpositive/correlationoptimizer10.q.out +++ b/ql/src/test/results/clientpositive/correlationoptimizer10.q.out @@ -432,24 +432,24 @@ STAGE PLANS: alias: x Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key is not null and (key < 200)) and (key > 20)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 200) and (key > 20)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key is not null and (key > 20)) and (key < 200)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > 20) and (key < 200)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -458,16 +458,16 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 29 Data size: 314 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 60 Data size: 642 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 29 Data size: 314 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 60 Data size: 642 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 29 Data size: 314 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 60 Data size: 642 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -483,7 +483,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 314 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 60 Data size: 642 Basic stats: COMPLETE Column stats: NONE TableScan alias: xx Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE @@ -504,14 +504,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 345 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 66 Data size: 706 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 31 Data size: 345 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 66 Data size: 706 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 31 Data size: 345 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 66 Data size: 706 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -589,24 +589,24 @@ STAGE PLANS: alias: x Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key is not null and (key < 200)) and (key > 20)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 200) and (key > 20)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key is not null and (key > 20)) and (key < 200)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > 20) and (key < 200)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE TableScan alias: xx Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE @@ -621,9 +621,9 @@ STAGE PLANS: value expressions: value (type: string) Reduce Operator Tree: Demux Operator - Statistics: Num rows: 67 Data size: 671 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 123 Data size: 1267 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 67 Data size: 671 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 123 Data size: 1267 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Left Semi Join 0 to 1 @@ -656,7 +656,7 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Mux Operator - Statistics: Num rows: 67 Data size: 671 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 123 Data size: 1267 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Left Semi Join 0 to 1 @@ -756,24 +756,24 @@ STAGE PLANS: alias: x Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key is not null and (key < 200)) and (key > 180)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 200) and (key > 180)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key is not null and (key < 200)) and (key > 180)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 200) and (key > 180)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -782,16 +782,16 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 29 Data size: 314 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 60 Data size: 642 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 29 Data size: 314 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 60 Data size: 642 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 29 Data size: 314 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 60 Data size: 642 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -807,7 +807,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 29 Data size: 314 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 60 Data size: 642 Basic stats: COMPLETE Column stats: NONE TableScan alias: xx Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE @@ -925,24 +925,24 @@ STAGE PLANS: alias: x Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key is not null and (key < 200)) and (key > 180)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 200) and (key > 180)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE TableScan alias: y Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key is not null and (key < 200)) and (key > 180)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 200) and (key > 180)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE TableScan alias: xx Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE @@ -957,9 +957,9 @@ STAGE PLANS: value expressions: value (type: string) Reduce Operator Tree: Demux Operator - Statistics: Num rows: 304 Data size: 3228 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 360 Data size: 3824 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 304 Data size: 3228 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 360 Data size: 3824 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Left Semi Join 0 to 1 @@ -992,7 +992,7 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Mux Operator - Statistics: Num rows: 304 Data size: 3228 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 360 Data size: 3824 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Left Semi Join 0 to 1 diff --git a/ql/src/test/results/clientpositive/correlationoptimizer13.q.out b/ql/src/test/results/clientpositive/correlationoptimizer13.q.out index f2a217b..29001f1 100644 --- a/ql/src/test/results/clientpositive/correlationoptimizer13.q.out +++ b/ql/src/test/results/clientpositive/correlationoptimizer13.q.out @@ -112,7 +112,7 @@ STAGE PLANS: key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: string) - Statistics: Num rows: 43 Data size: 960 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 85 Data size: 1898 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Reduce Operator Tree: Join Operator @@ -122,11 +122,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {KEY.reducesinkkey1} {VALUE._col0} 1 {KEY.reducesinkkey0} {KEY.reducesinkkey1} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 47 Data size: 1056 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 93 Data size: 2087 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: string), _col2 (type: bigint), _col5 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 47 Data size: 1056 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 93 Data size: 2087 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -141,15 +141,15 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: string), _col4 (type: bigint), _col5 (type: bigint) sort order: ++++++ - Statistics: Num rows: 47 Data size: 1056 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 93 Data size: 2087 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: int), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: bigint), KEY.reducesinkkey5 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 47 Data size: 1056 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 93 Data size: 2087 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 47 Data size: 1056 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 93 Data size: 2087 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -162,23 +162,23 @@ STAGE PLANS: alias: x Statistics: Num rows: 1028 Data size: 22964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((c1 < 120) and c1 is not null) and c3 is not null) (type: boolean) - Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE + predicate: ((c1 < 120) and c3 is not null) (type: boolean) + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: c3 (type: string), c1 (type: int) outputColumnNames: c3, c1 - Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: c3 (type: string), c1 (type: int) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: int) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) - Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Reduce Operator Tree: Group By Operator @@ -186,11 +186,11 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 43 Data size: 960 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 85 Data size: 1898 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int), _col0 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 43 Data size: 960 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 85 Data size: 1898 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: diff --git a/ql/src/test/results/clientpositive/correlationoptimizer8.q.out b/ql/src/test/results/clientpositive/correlationoptimizer8.q.out index 9e34fe1..3c737d5 100644 --- a/ql/src/test/results/clientpositive/correlationoptimizer8.q.out +++ b/ql/src/test/results/clientpositive/correlationoptimizer8.q.out @@ -46,23 +46,23 @@ STAGE PLANS: alias: x Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 20) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 20) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: key - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -70,11 +70,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -87,21 +87,21 @@ STAGE PLANS: Map Operator Tree: TableScan Union - Statistics: Num rows: 82 Data size: 870 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 82 Data size: 870 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) TableScan Union - Statistics: Num rows: 82 Data size: 870 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 82 Data size: 870 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) TableScan alias: x @@ -123,14 +123,14 @@ STAGE PLANS: 0 {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col1, _col2, _col3 - Statistics: Num rows: 90 Data size: 957 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1938 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col3 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 90 Data size: 957 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1938 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 90 Data size: 957 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1938 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -143,23 +143,23 @@ STAGE PLANS: alias: x1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > 100) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key > 100) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: key - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -167,11 +167,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -250,45 +250,45 @@ STAGE PLANS: alias: x Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 20) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 20) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: key - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) TableScan alias: x1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > 100) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key > 100) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: key - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) TableScan alias: x @@ -304,21 +304,21 @@ STAGE PLANS: value expressions: value (type: string) Reduce Operator Tree: Demux Operator - Statistics: Num rows: 179 Data size: 1861 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 345 Data size: 3625 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 89 Data size: 925 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 172 Data size: 1807 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 89 Data size: 925 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 172 Data size: 1807 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 178 Data size: 1850 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 344 Data size: 3614 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 357 Data size: 3711 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 689 Data size: 7239 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -343,15 +343,15 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 89 Data size: 925 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 172 Data size: 1807 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 89 Data size: 925 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 172 Data size: 1807 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 178 Data size: 1850 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 344 Data size: 3614 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 357 Data size: 3711 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 689 Data size: 7239 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -372,7 +372,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Mux Operator - Statistics: Num rows: 357 Data size: 3711 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 689 Data size: 7239 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -938,23 +938,23 @@ STAGE PLANS: alias: x Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 20) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 20) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: key - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -962,11 +962,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -979,21 +979,21 @@ STAGE PLANS: Map Operator Tree: TableScan Union - Statistics: Num rows: 82 Data size: 870 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 82 Data size: 870 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) TableScan Union - Statistics: Num rows: 82 Data size: 870 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 82 Data size: 870 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) TableScan alias: x @@ -1015,14 +1015,14 @@ STAGE PLANS: 0 {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col1, _col2, _col3 - Statistics: Num rows: 90 Data size: 957 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1938 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col3 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 90 Data size: 957 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1938 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 90 Data size: 957 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1938 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1035,23 +1035,23 @@ STAGE PLANS: alias: x1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > 100) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key > 100) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: key, value - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: key (type: string), value (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Reduce Operator Tree: Group By Operator @@ -1059,11 +1059,11 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: diff --git a/ql/src/test/results/clientpositive/correlationoptimizer9.q.out b/ql/src/test/results/clientpositive/correlationoptimizer9.q.out index 38da3c3..864bb21 100644 --- a/ql/src/test/results/clientpositive/correlationoptimizer9.q.out +++ b/ql/src/test/results/clientpositive/correlationoptimizer9.q.out @@ -54,23 +54,23 @@ STAGE PLANS: alias: x1 Statistics: Num rows: 1028 Data size: 22964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((c2 > 100) and c2 is not null) (type: boolean) - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + predicate: (c2 > 100) (type: boolean) + Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: c2 (type: int) outputColumnNames: c2 - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: c2 (type: int) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -78,11 +78,11 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 85 Data size: 1898 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 85 Data size: 1898 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -98,14 +98,14 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 85 Data size: 1898 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) TableScan Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 85 Data size: 1898 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Join Operator @@ -115,14 +115,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 93 Data size: 2087 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 188 Data size: 4200 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col2 (type: int), _col1 (type: bigint), _col3 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 93 Data size: 2087 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 188 Data size: 4200 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 93 Data size: 2087 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 188 Data size: 4200 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -135,23 +135,23 @@ STAGE PLANS: alias: x Statistics: Num rows: 1028 Data size: 22964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((c1 < 120) and c1 is not null) (type: boolean) - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + predicate: (c1 < 120) (type: boolean) + Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: c1 (type: int) outputColumnNames: c1 - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: c1 (type: int) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -159,11 +159,11 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 85 Data size: 1898 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 85 Data size: 1898 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -234,61 +234,61 @@ STAGE PLANS: alias: x1 Statistics: Num rows: 1028 Data size: 22964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((c2 > 100) and c2 is not null) (type: boolean) - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + predicate: (c2 > 100) (type: boolean) + Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: c2 (type: int) outputColumnNames: c2 - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: c2 (type: int) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) TableScan alias: x Statistics: Num rows: 1028 Data size: 22964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((c1 < 120) and c1 is not null) (type: boolean) - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + predicate: (c1 < 120) (type: boolean) + Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: c1 (type: int) outputColumnNames: c1 - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: c1 (type: int) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Demux Operator - Statistics: Num rows: 342 Data size: 7638 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 684 Data size: 15278 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 342 Data size: 7638 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 684 Data size: 15278 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -313,13 +313,13 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 342 Data size: 7639 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 342 Data size: 7638 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 684 Data size: 15278 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -454,7 +454,7 @@ STAGE PLANS: key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: string) - Statistics: Num rows: 43 Data size: 960 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 85 Data size: 1898 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Reduce Operator Tree: Join Operator @@ -464,14 +464,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {KEY.reducesinkkey1} {VALUE._col0} 1 {KEY.reducesinkkey0} {KEY.reducesinkkey1} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 47 Data size: 1056 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 93 Data size: 2087 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col3 (type: int), _col4 (type: string), _col2 (type: bigint), _col5 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 47 Data size: 1056 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 93 Data size: 2087 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 47 Data size: 1056 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 93 Data size: 2087 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -484,23 +484,23 @@ STAGE PLANS: alias: x Statistics: Num rows: 1028 Data size: 22964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((c1 < 120) and c1 is not null) and c3 is not null) (type: boolean) - Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE + predicate: ((c1 < 120) and c3 is not null) (type: boolean) + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: c1 (type: int), c3 (type: string) outputColumnNames: c1, c3 - Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: c1 (type: int), c3 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: string) - Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Reduce Operator Tree: Group By Operator @@ -508,11 +508,11 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 43 Data size: 960 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 85 Data size: 1898 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 43 Data size: 960 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 85 Data size: 1898 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -603,39 +603,39 @@ STAGE PLANS: alias: x Statistics: Num rows: 1028 Data size: 22964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((c1 < 120) and c1 is not null) and c3 is not null) (type: boolean) - Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE + predicate: ((c1 < 120) and c3 is not null) (type: boolean) + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: c1 (type: int), c3 (type: string) outputColumnNames: c1, c3 - Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: c1 (type: int), c3 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: string) - Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 171 Data size: 3819 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Reduce Operator Tree: Demux Operator - Statistics: Num rows: 172 Data size: 3842 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 257 Data size: 5740 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: int), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 128 Data size: 2858 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 128 Data size: 2858 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 172 Data size: 3842 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 256 Data size: 5716 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -660,13 +660,13 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 128 Data size: 2858 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 86 Data size: 1921 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 128 Data size: 2858 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 172 Data size: 3842 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 256 Data size: 5716 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 diff --git a/ql/src/test/results/clientpositive/create_like.q.out b/ql/src/test/results/clientpositive/create_like.q.out index a3d0980..7d05b4f 100644 --- a/ql/src/test/results/clientpositive/create_like.q.out +++ b/ql/src/test/results/clientpositive/create_like.q.out @@ -265,3 +265,143 @@ POSTHOOK: Input: default@table4 4 5 6 +PREHOOK: query: CREATE TABLE doctors STORED AS AVRO TBLPROPERTIES ('avro.schema.literal'='{ + "namespace": "testing.hive.avro.serde", + "name": "doctors", + "type": "record", + "fields": [ + { + "name":"number", + "type":"int", + "doc":"Order of playing the role" + }, + { + "name":"first_name", + "type":"string", + "doc":"first name of actor playing role" + }, + { + "name":"last_name", + "type":"string", + "doc":"last name of actor playing role" + } + ] +}') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@doctors +POSTHOOK: query: CREATE TABLE doctors STORED AS AVRO TBLPROPERTIES ('avro.schema.literal'='{ + "namespace": "testing.hive.avro.serde", + "name": "doctors", + "type": "record", + "fields": [ + { + "name":"number", + "type":"int", + "doc":"Order of playing the role" + }, + { + "name":"first_name", + "type":"string", + "doc":"first name of actor playing role" + }, + { + "name":"last_name", + "type":"string", + "doc":"last name of actor playing role" + } + ] +}') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@doctors +PREHOOK: query: alter table doctors set tblproperties ('k1'='v1', 'k2'='v2') +PREHOOK: type: ALTERTABLE_PROPERTIES +PREHOOK: Input: default@doctors +PREHOOK: Output: default@doctors +POSTHOOK: query: alter table doctors set tblproperties ('k1'='v1', 'k2'='v2') +POSTHOOK: type: ALTERTABLE_PROPERTIES +POSTHOOK: Input: default@doctors +POSTHOOK: Output: default@doctors +PREHOOK: query: DESCRIBE FORMATTED doctors +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@doctors +POSTHOOK: query: DESCRIBE FORMATTED doctors +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@doctors +# col_name data_type comment + +number int from deserializer +first_name string from deserializer +last_name string from deserializer + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE false + avro.schema.literal {\n \"namespace\": \"testing.hive.avro.serde\",\n \"name\": \"doctors\",\n \"type\": \"record\",\n \"fields\": [\n {\n \"name\":\"number\",\n \"type\":\"int\",\n \"doc\":\"Order of playing the role\"\n },\n {\n \"name\":\"first_name\",\n \"type\":\"string\",\n \"doc\":\"first name of actor playing role\"\n },\n {\n \"name\":\"last_name\",\n \"type\":\"string\",\n \"doc\":\"last name of actor playing role\"\n }\n ]\n} + k1 v1 + k2 v2 +#### A masked pattern was here #### + numFiles 0 + numRows -1 + rawDataSize -1 + totalSize 0 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.avro.AvroSerDe +InputFormat: org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: CREATE TABLE doctors2 like doctors +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@doctors2 +POSTHOOK: query: CREATE TABLE doctors2 like doctors +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@doctors2 +PREHOOK: query: DESCRIBE FORMATTED doctors2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@doctors2 +POSTHOOK: query: DESCRIBE FORMATTED doctors2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@doctors2 +# col_name data_type comment + +number int from deserializer +first_name string from deserializer +last_name string from deserializer + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + avro.schema.literal {\n \"namespace\": \"testing.hive.avro.serde\",\n \"name\": \"doctors\",\n \"type\": \"record\",\n \"fields\": [\n {\n \"name\":\"number\",\n \"type\":\"int\",\n \"doc\":\"Order of playing the role\"\n },\n {\n \"name\":\"first_name\",\n \"type\":\"string\",\n \"doc\":\"first name of actor playing role\"\n },\n {\n \"name\":\"last_name\",\n \"type\":\"string\",\n \"doc\":\"last name of actor playing role\"\n }\n ]\n} +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.avro.AvroSerDe +InputFormat: org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 diff --git a/ql/src/test/results/clientpositive/cross_product_check_2.q.out b/ql/src/test/results/clientpositive/cross_product_check_2.q.out index 32d94a6..7cf3e9c 100644 --- a/ql/src/test/results/clientpositive/cross_product_check_2.q.out +++ b/ql/src/test/results/clientpositive/cross_product_check_2.q.out @@ -189,7 +189,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[26][bigTable=?] in task 'Stage-6:MAPRED' is a cross product +Warning: Map Join MAPJOIN[25][bigTable=?] in task 'Stage-6:MAPRED' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 on d1.key = d2.key @@ -337,8 +337,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[22][bigTable=?] in task 'Stage-6:MAPRED' is a cross product -Warning: Map Join MAPJOIN[23][bigTable=d2] in task 'Stage-2:MAPRED' is a cross product +Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Stage-6:MAPRED' is a cross product +Warning: Map Join MAPJOIN[22][bigTable=d2] in task 'Stage-2:MAPRED' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 PREHOOK: type: QUERY POSTHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 @@ -474,8 +474,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[100][bigTable=?] in task 'Stage-7:MAPRED' is a cross product -Warning: Map Join MAPJOIN[66][bigTable=?] in task 'Stage-6:MAPRED' is a cross product +Warning: Map Join MAPJOIN[97][bigTable=?] in task 'Stage-7:MAPRED' is a cross product +Warning: Map Join MAPJOIN[64][bigTable=?] in task 'Stage-6:MAPRED' is a cross product Warning: Shuffle Join JOIN[21][tables = [ss, od1]] in Stage 'Stage-3:MAPRED' is a cross product PREHOOK: query: explain select * from (select A.key from A group by key) ss join diff --git a/ql/src/test/results/clientpositive/ctas_colname.q.out b/ql/src/test/results/clientpositive/ctas_colname.q.out index 97dacf6..c26a66d 100644 --- a/ql/src/test/results/clientpositive/ctas_colname.q.out +++ b/ql/src/test/results/clientpositive/ctas_colname.q.out @@ -3,16 +3,20 @@ PREHOOK: query: -- SORT_QUERY_RESULTS -- HIVE-4392, column aliases from expressionRR (GBY, etc.) are not valid name for table -- group by + + explain -create table summary as select *, sum(key), count(value) from src +create table summary as select *, key + 1, concat(value, value) from src limit 20 PREHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: query: -- SORT_QUERY_RESULTS -- HIVE-4392, column aliases from expressionRR (GBY, etc.) are not valid name for table -- group by + + explain -create table summary as select *, sum(key), count(value) from src +create table summary as select *, key + 1, concat(value, value) from src limit 20 POSTHOOK: type: CREATETABLE_AS_SELECT STAGE DEPENDENCIES: Stage-1 is a root stage @@ -28,31 +32,27 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value + expressions: key (type: string), value (type: string), (key + 1) (type: double), concat(value, value) (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: sum(key), count(value) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 20 + Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: double), _col1 (type: bigint) + Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double), _col3 (type: string) Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0), count(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: double), _col1 (type: bigint), _col0 (type: double), _col1 (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: double), VALUE._col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 20 + Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -68,7 +68,7 @@ STAGE PLANS: Stage: Stage-3 Create Table Operator: Create Table - columns: _col0 double, _col1 bigint, _c1 double, _c2 bigint + columns: key string, value string, _c1 double, _c2 string input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -77,12 +77,12 @@ STAGE PLANS: Stage: Stage-2 Stats-Aggr Operator -PREHOOK: query: create table summary as select *, sum(key), count(value) from src +PREHOOK: query: create table summary as select *, key + 1, concat(value, value) from src limit 20 PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@src PREHOOK: Output: database:default PREHOOK: Output: default@summary -POSTHOOK: query: create table summary as select *, sum(key), count(value) from src +POSTHOOK: query: create table summary as select *, key + 1, concat(value, value) from src limit 20 POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@src POSTHOOK: Output: database:default @@ -95,10 +95,10 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@summary # col_name data_type comment -_col0 double -_col1 bigint +key string +value string _c1 double -_c2 bigint +_c2 string # Detailed Table Information Database: default @@ -110,9 +110,9 @@ Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE true numFiles 1 - numRows 1 - rawDataSize 25 - totalSize 26 + numRows 20 + rawDataSize 620 + totalSize 640 #### A masked pattern was here #### # Storage Information @@ -133,7 +133,26 @@ POSTHOOK: query: select * from summary POSTHOOK: type: QUERY POSTHOOK: Input: default@summary #### A masked pattern was here #### -130091.0 500 130091.0 500 +128 val_128 129.0 val_128val_128 +150 val_150 151.0 val_150val_150 +165 val_165 166.0 val_165val_165 +193 val_193 194.0 val_193val_193 +213 val_213 214.0 val_213val_213 +224 val_224 225.0 val_224val_224 +238 val_238 239.0 val_238val_238 +255 val_255 256.0 val_255val_255 +265 val_265 266.0 val_265val_265 +27 val_27 28.0 val_27val_27 +273 val_273 274.0 val_273val_273 +278 val_278 279.0 val_278val_278 +311 val_311 312.0 val_311val_311 +369 val_369 370.0 val_369val_369 +401 val_401 402.0 val_401val_401 +409 val_409 410.0 val_409val_409 +484 val_484 485.0 val_484val_484 +66 val_66 67.0 val_66val_66 +86 val_86 87.0 val_86val_86 +98 val_98 99.0 val_98val_98 PREHOOK: query: -- window functions explain create table x4 as select *, rank() over(partition by key order by value) as rr from src1 diff --git a/ql/src/test/results/clientpositive/decimal_serde.q.out b/ql/src/test/results/clientpositive/decimal_serde.q.out index e461c2e..d651799 100644 --- a/ql/src/test/results/clientpositive/decimal_serde.q.out +++ b/ql/src/test/results/clientpositive/decimal_serde.q.out @@ -96,6 +96,42 @@ POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@decimal_text POSTHOOK: Output: database:default POSTHOOK: Output: default@DECIMAL_RC +PREHOOK: query: describe formatted DECIMAL_RC +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@decimal_rc +POSTHOOK: query: describe formatted DECIMAL_RC +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@decimal_rc +# col_name data_type comment + +key decimal(10,0) +value int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 38 + rawDataSize 157 + totalSize 278 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 PREHOOK: query: CREATE TABLE DECIMAL_LAZY_COL ROW FORMAT SERDE "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe" STORED AS RCFile AS @@ -112,6 +148,42 @@ POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@decimal_rc POSTHOOK: Output: database:default POSTHOOK: Output: default@DECIMAL_LAZY_COL +PREHOOK: query: describe formatted DECIMAL_LAZY_COL +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@decimal_lazy_col +POSTHOOK: query: describe formatted DECIMAL_LAZY_COL +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@decimal_lazy_col +# col_name data_type comment + +key decimal(10,0) +value int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE true + numFiles 1 + numRows 38 + rawDataSize 157 + totalSize 278 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 PREHOOK: query: CREATE TABLE DECIMAL_SEQUENCE ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' diff --git a/ql/src/test/results/clientpositive/decimal_udf.q.out b/ql/src/test/results/clientpositive/decimal_udf.q.out index 8fda14c..6bd876a 100644 --- a/ql/src/test/results/clientpositive/decimal_udf.q.out +++ b/ql/src/test/results/clientpositive/decimal_udf.q.out @@ -973,12 +973,12 @@ STAGE PLANS: alias: decimal_udf Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key <> 0)) (type: boolean) - Statistics: Num rows: 2 Data size: 239 Basic stats: COMPLETE Column stats: NONE + predicate: (key <> 0) (type: boolean) + Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: (key / key) (type: decimal(38,24)) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 239 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: SELECT key / key FROM DECIMAL_UDF WHERE key is not null and key <> 0 @@ -1039,12 +1039,12 @@ STAGE PLANS: alias: decimal_udf Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (value is not null and (value <> 0)) (type: boolean) - Statistics: Num rows: 2 Data size: 239 Basic stats: COMPLETE Column stats: NONE + predicate: (value <> 0) (type: boolean) + Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: (key / value) (type: decimal(31,21)) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 239 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: SELECT key / value FROM DECIMAL_UDF WHERE value is not null and value <> 0 @@ -1095,12 +1095,12 @@ STAGE PLANS: alias: decimal_udf Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (value is not null and (value <> 0)) (type: boolean) - Statistics: Num rows: 2 Data size: 239 Basic stats: COMPLETE Column stats: NONE + predicate: (value <> 0) (type: boolean) + Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: (key / (value / 2)) (type: double) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 239 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: SELECT key / (value/2) FROM DECIMAL_UDF WHERE value is not null and value <> 0 diff --git a/ql/src/test/results/clientpositive/delete_all_partitioned.q.out b/ql/src/test/results/clientpositive/delete_all_partitioned.q.out index c5149b2..90f8753 100644 --- a/ql/src/test/results/clientpositive/delete_all_partitioned.q.out +++ b/ql/src/test/results/clientpositive/delete_all_partitioned.q.out @@ -84,5 +84,3 @@ POSTHOOK: Input: default@acid_dap POSTHOOK: Input: default@acid_dap@ds=today POSTHOOK: Input: default@acid_dap@ds=tomorrow #### A masked pattern was here #### --1071480828 aw724t8c5558x2xneC624 today --1072076362 2uLyD28144vklju213J1mr today diff --git a/ql/src/test/results/clientpositive/groupby_grouping_sets4.q.out b/ql/src/test/results/clientpositive/groupby_grouping_sets4.q.out index c545506..3bbd36f 100644 --- a/ql/src/test/results/clientpositive/groupby_grouping_sets4.q.out +++ b/ql/src/test/results/clientpositive/groupby_grouping_sets4.q.out @@ -52,7 +52,7 @@ STAGE PLANS: alias: t1 Statistics: Num rows: 0 Data size: 36 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: ((a < 3) and a is not null) (type: boolean) + predicate: (a < 3) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: a (type: string), b (type: string) @@ -133,7 +133,7 @@ STAGE PLANS: alias: t1 Statistics: Num rows: 0 Data size: 36 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: ((a < 3) and a is not null) (type: boolean) + predicate: (a < 3) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: a (type: string), b (type: string) @@ -238,7 +238,7 @@ STAGE PLANS: alias: t1 Statistics: Num rows: 0 Data size: 36 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: ((a < 3) and a is not null) (type: boolean) + predicate: (a < 3) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: a (type: string), b (type: string) @@ -343,7 +343,7 @@ STAGE PLANS: alias: t1 Statistics: Num rows: 0 Data size: 36 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: ((a < 3) and a is not null) (type: boolean) + predicate: (a < 3) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: a (type: string), b (type: string) diff --git a/ql/src/test/results/clientpositive/groupby_ppd.q.out b/ql/src/test/results/clientpositive/groupby_ppd.q.out index a086527..403ebd3 100644 --- a/ql/src/test/results/clientpositive/groupby_ppd.q.out +++ b/ql/src/test/results/clientpositive/groupby_ppd.q.out @@ -79,7 +79,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: int) + expressions: 1 (type: int), _col1 (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/index_auth.q.out b/ql/src/test/results/clientpositive/index_auth.q.out index 385b639..39a1096 100644 --- a/ql/src/test/results/clientpositive/index_auth.q.out +++ b/ql/src/test/results/clientpositive/index_auth.q.out @@ -24,7 +24,7 @@ PREHOOK: query: SHOW INDEXES ON foobar PREHOOK: type: SHOWINDEXES POSTHOOK: query: SHOW INDEXES ON foobar POSTHOOK: type: SHOWINDEXES -srcpart_auth_index foobar key default__foobar_srcpart_auth_index__ bitmap +srcpart_auth_index foobar key default.default__foobar_srcpart_auth_index__ bitmap PREHOOK: query: grant select on table foobar to user hive_test_user PREHOOK: type: GRANT_PRIVILEGE PREHOOK: Output: default@foobar diff --git a/ql/src/test/results/clientpositive/index_auto.q.out b/ql/src/test/results/clientpositive/index_auto.q.out index d1f5fa2..124a9d2 100644 --- a/ql/src/test/results/clientpositive/index_auto.q.out +++ b/ql/src/test/results/clientpositive/index_auto.q.out @@ -146,7 +146,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__src_src_index__ + alias: default.default__src_src_index__ filterExpr: ((key > 80) and (key < 100)) (type: boolean) Filter Operator predicate: ((key > 80) and (key < 100)) (type: boolean) diff --git a/ql/src/test/results/clientpositive/index_auto_file_format.q.out b/ql/src/test/results/clientpositive/index_auto_file_format.q.out index eca656f..85a57f3 100644 --- a/ql/src/test/results/clientpositive/index_auto_file_format.q.out +++ b/ql/src/test/results/clientpositive/index_auto_file_format.q.out @@ -40,7 +40,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__src_src_index__ + alias: default.default__src_src_index__ filterExpr: (key = 86) (type: boolean) Filter Operator predicate: (key = 86) (type: boolean) @@ -156,7 +156,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__src_src_index__ + alias: default.default__src_src_index__ filterExpr: (key = 86) (type: boolean) Filter Operator predicate: (key = 86) (type: boolean) diff --git a/ql/src/test/results/clientpositive/index_auto_mult_tables.q.out b/ql/src/test/results/clientpositive/index_auto_mult_tables.q.out index ab0f06f..47fed0c 100644 --- a/ql/src/test/results/clientpositive/index_auto_mult_tables.q.out +++ b/ql/src/test/results/clientpositive/index_auto_mult_tables.q.out @@ -22,24 +22,24 @@ STAGE PLANS: alias: b Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((key is not null and (key > 70)) and (key < 90)) and (key > 80)) and (key < 100)) (type: boolean) - Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE + predicate: ((((key > 70) and (key < 90)) and (key > 80)) and (key < 100)) (type: boolean) + Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((key is not null and (key > 80)) and (key < 100)) and (key > 70)) and (key < 90)) (type: boolean) - Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + predicate: ((((key > 80) and (key < 100)) and (key > 70)) and (key < 90)) (type: boolean) + Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Reduce Operator Tree: Join Operator @@ -49,14 +49,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 139 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 279 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 139 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 279 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 13 Data size: 139 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 279 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -203,7 +203,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__srcpart_srcpart_index__ + alias: default.default__srcpart_srcpart_index__ filterExpr: (((((key > 70) and (key < 90)) and (key > 80)) and (key < 100)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean) Filter Operator predicate: (((((key > 70) and (key < 90)) and (key > 80)) and (key < 100)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean) @@ -247,28 +247,28 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b - filterExpr: ((((key is not null and (key > 70)) and (key < 90)) and (key > 80)) and (key < 100)) (type: boolean) + filterExpr: ((((key > 70) and (key < 90)) and (key > 80)) and (key < 100)) (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((key is not null and (key > 70)) and (key < 90)) and (key > 80)) and (key < 100)) (type: boolean) - Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE + predicate: ((((key > 70) and (key < 90)) and (key > 80)) and (key < 100)) (type: boolean) + Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE TableScan alias: a - filterExpr: ((((key is not null and (key > 80)) and (key < 100)) and (key > 70)) and (key < 90)) (type: boolean) + filterExpr: ((((key > 80) and (key < 100)) and (key > 70)) and (key < 90)) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((key is not null and (key > 80)) and (key < 100)) and (key > 70)) and (key < 90)) (type: boolean) - Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + predicate: ((((key > 80) and (key < 100)) and (key > 70)) and (key < 90)) (type: boolean) + Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Reduce Operator Tree: Join Operator @@ -278,14 +278,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 139 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 279 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 139 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 279 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 13 Data size: 139 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 279 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -295,7 +295,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__src_src_index__ + alias: default.default__src_src_index__ filterExpr: (((((key > 80) and (key < 100)) and (key > 70)) and (key < 90)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean) Filter Operator predicate: (((((key > 80) and (key < 100)) and (key > 70)) and (key < 90)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean) diff --git a/ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out b/ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out index 20fa238..1e344fa 100644 --- a/ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out +++ b/ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out @@ -22,24 +22,24 @@ STAGE PLANS: alias: b Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((key is not null and (key > 70)) and (key < 90)) and (key > 80)) and (key < 100)) (type: boolean) - Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE + predicate: ((((key > 70) and (key < 90)) and (key > 80)) and (key < 100)) (type: boolean) + Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((key is not null and (key > 80)) and (key < 100)) and (key > 70)) and (key < 90)) (type: boolean) - Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + predicate: ((((key > 80) and (key < 100)) and (key > 70)) and (key < 90)) (type: boolean) + Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Reduce Operator Tree: Join Operator @@ -49,14 +49,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 139 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 279 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 139 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 279 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 13 Data size: 139 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 279 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -213,7 +213,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__srcpart_srcpart_index__ + alias: default.default__srcpart_srcpart_index__ filterExpr: ((((key > 70) and (key < 90)) and (key > 80)) and (key < 100)) (type: boolean) Filter Operator predicate: ((((key > 70) and (key < 90)) and (key > 80)) and (key < 100)) (type: boolean) @@ -247,28 +247,28 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b - filterExpr: ((((key is not null and (key > 70)) and (key < 90)) and (key > 80)) and (key < 100)) (type: boolean) + filterExpr: ((((key > 70) and (key < 90)) and (key > 80)) and (key < 100)) (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((key is not null and (key > 70)) and (key < 90)) and (key > 80)) and (key < 100)) (type: boolean) - Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE + predicate: ((((key > 70) and (key < 90)) and (key > 80)) and (key < 100)) (type: boolean) + Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE TableScan alias: a - filterExpr: ((((key is not null and (key > 80)) and (key < 100)) and (key > 70)) and (key < 90)) (type: boolean) + filterExpr: ((((key > 80) and (key < 100)) and (key > 70)) and (key < 90)) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((key is not null and (key > 80)) and (key < 100)) and (key > 70)) and (key < 90)) (type: boolean) - Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + predicate: ((((key > 80) and (key < 100)) and (key > 70)) and (key < 90)) (type: boolean) + Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Reduce Operator Tree: Join Operator @@ -278,14 +278,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 139 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 279 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 139 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 279 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 13 Data size: 139 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 279 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -323,7 +323,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__src_src_index__ + alias: default.default__src_src_index__ filterExpr: ((((key > 80) and (key < 100)) and (key > 70)) and (key < 90)) (type: boolean) Filter Operator predicate: ((((key > 80) and (key < 100)) and (key > 70)) and (key < 90)) (type: boolean) diff --git a/ql/src/test/results/clientpositive/index_auto_multiple.q.out b/ql/src/test/results/clientpositive/index_auto_multiple.q.out index 2408d48..722ee6e 100644 --- a/ql/src/test/results/clientpositive/index_auto_multiple.q.out +++ b/ql/src/test/results/clientpositive/index_auto_multiple.q.out @@ -60,7 +60,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__src_src_key_index__ + alias: default.default__src_src_key_index__ filterExpr: (key = 86) (type: boolean) Filter Operator predicate: (key = 86) (type: boolean) diff --git a/ql/src/test/results/clientpositive/index_auto_partitioned.q.out b/ql/src/test/results/clientpositive/index_auto_partitioned.q.out index da57bea..4777c53 100644 --- a/ql/src/test/results/clientpositive/index_auto_partitioned.q.out +++ b/ql/src/test/results/clientpositive/index_auto_partitioned.q.out @@ -54,7 +54,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__srcpart_src_part_index__ + alias: default.default__srcpart_src_part_index__ filterExpr: ((key = 86) and (ds = '2008-04-09')) (type: boolean) Filter Operator predicate: (key = 86) (type: boolean) diff --git a/ql/src/test/results/clientpositive/index_auto_self_join.q.out b/ql/src/test/results/clientpositive/index_auto_self_join.q.out index 94ded26..2bc80f4 100644 --- a/ql/src/test/results/clientpositive/index_auto_self_join.q.out +++ b/ql/src/test/results/clientpositive/index_auto_self_join.q.out @@ -123,7 +123,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__src_src_index__ + alias: default.default__src_src_index__ filterExpr: (((key > 70) and (key < 90)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean) Filter Operator predicate: (((key > 70) and (key < 90)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean) @@ -216,7 +216,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__src_src_index__ + alias: default.default__src_src_index__ filterExpr: (((key > 80) and (key < 100)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean) Filter Operator predicate: (((key > 80) and (key < 100)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean) diff --git a/ql/src/test/results/clientpositive/index_auto_unused.q.out b/ql/src/test/results/clientpositive/index_auto_unused.q.out index 8ebfcae..ad799e8 100644 --- a/ql/src/test/results/clientpositive/index_auto_unused.q.out +++ b/ql/src/test/results/clientpositive/index_auto_unused.q.out @@ -371,7 +371,7 @@ STAGE PLANS: predicate: (key < 10) (type: boolean) Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) + expressions: key (type: string), value (type: string), '2008-04-09' (type: string), '12' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/index_auto_update.q.out b/ql/src/test/results/clientpositive/index_auto_update.q.out index 85f26ac..0f996b5 100644 --- a/ql/src/test/results/clientpositive/index_auto_update.q.out +++ b/ql/src/test/results/clientpositive/index_auto_update.q.out @@ -222,7 +222,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__temp_temp_index__ + alias: default.default__temp_temp_index__ filterExpr: (key = 86) (type: boolean) Filter Operator predicate: (key = 86) (type: boolean) diff --git a/ql/src/test/results/clientpositive/index_bitmap_auto_partitioned.q.out b/ql/src/test/results/clientpositive/index_bitmap_auto_partitioned.q.out index 473df0a..052acbb 100644 --- a/ql/src/test/results/clientpositive/index_bitmap_auto_partitioned.q.out +++ b/ql/src/test/results/clientpositive/index_bitmap_auto_partitioned.q.out @@ -52,7 +52,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__srcpart_src_part_index__ + alias: default.default__srcpart_src_part_index__ filterExpr: ((key = 86) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean) Filter Operator predicate: ((key = 86) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean) diff --git a/ql/src/test/results/clientpositive/index_bitmap_compression.q.out b/ql/src/test/results/clientpositive/index_bitmap_compression.q.out index b5a172f..f136259 100644 --- a/ql/src/test/results/clientpositive/index_bitmap_compression.q.out +++ b/ql/src/test/results/clientpositive/index_bitmap_compression.q.out @@ -38,7 +38,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__src_src_index__ + alias: default.default__src_src_index__ filterExpr: (((key > 80) and (key < 100)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean) Filter Operator predicate: (((key > 80) and (key < 100)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean) diff --git a/ql/src/test/results/clientpositive/index_compression.q.out b/ql/src/test/results/clientpositive/index_compression.q.out index 241c2e9..b44bce8 100644 --- a/ql/src/test/results/clientpositive/index_compression.q.out +++ b/ql/src/test/results/clientpositive/index_compression.q.out @@ -42,7 +42,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__src_src_index__ + alias: default.default__src_src_index__ filterExpr: ((key > 80) and (key < 100)) (type: boolean) Filter Operator predicate: ((key > 80) and (key < 100)) (type: boolean) diff --git a/ql/src/test/results/clientpositive/index_in_db.q.out b/ql/src/test/results/clientpositive/index_in_db.q.out new file mode 100644 index 0000000..91af7ba --- /dev/null +++ b/ql/src/test/results/clientpositive/index_in_db.q.out @@ -0,0 +1,51 @@ +PREHOOK: query: drop database if exists index_test_db cascade +PREHOOK: type: DROPDATABASE +POSTHOOK: query: drop database if exists index_test_db cascade +POSTHOOK: type: DROPDATABASE +PREHOOK: query: -- Test selecting selecting from a table that is backed by an index +-- create table, index in a db, then set default db as current db, and try selecting + +create database index_test_db +PREHOOK: type: CREATEDATABASE +PREHOOK: Output: database:index_test_db +POSTHOOK: query: -- Test selecting selecting from a table that is backed by an index +-- create table, index in a db, then set default db as current db, and try selecting + +create database index_test_db +POSTHOOK: type: CREATEDATABASE +POSTHOOK: Output: database:index_test_db +PREHOOK: query: use index_test_db +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:index_test_db +POSTHOOK: query: use index_test_db +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:index_test_db +PREHOOK: query: create table testtb (id int, name string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:index_test_db +PREHOOK: Output: index_test_db@testtb +POSTHOOK: query: create table testtb (id int, name string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:index_test_db +POSTHOOK: Output: index_test_db@testtb +PREHOOK: query: create index id_index on table testtb (id) as 'COMPACT' WITH DEFERRED REBUILD in table testdb_id_idx_tb +PREHOOK: type: CREATEINDEX +PREHOOK: Input: index_test_db@testtb +POSTHOOK: query: create index id_index on table testtb (id) as 'COMPACT' WITH DEFERRED REBUILD in table testdb_id_idx_tb +POSTHOOK: type: CREATEINDEX +POSTHOOK: Input: index_test_db@testtb +POSTHOOK: Output: index_test_db@testdb_id_idx_tb +PREHOOK: query: use default +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:default +POSTHOOK: query: use default +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:default +PREHOOK: query: select * from index_test_db.testtb where id>2 +PREHOOK: type: QUERY +PREHOOK: Input: index_test_db@testtb +#### A masked pattern was here #### +POSTHOOK: query: select * from index_test_db.testtb where id>2 +POSTHOOK: type: QUERY +POSTHOOK: Input: index_test_db@testtb +#### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/index_serde.q.out b/ql/src/test/results/clientpositive/index_serde.q.out index 5b4cbfc..c0fc7e5 100644 --- a/ql/src/test/results/clientpositive/index_serde.q.out +++ b/ql/src/test/results/clientpositive/index_serde.q.out @@ -134,7 +134,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__doctors_doctors_index__ + alias: default.default__doctors_doctors_index__ filterExpr: (number > 6) (type: boolean) Filter Operator predicate: (number > 6) (type: boolean) diff --git a/ql/src/test/results/clientpositive/index_stale_partitioned.q.out b/ql/src/test/results/clientpositive/index_stale_partitioned.q.out index 5988d76..edd9746 100644 --- a/ql/src/test/results/clientpositive/index_stale_partitioned.q.out +++ b/ql/src/test/results/clientpositive/index_stale_partitioned.q.out @@ -95,7 +95,7 @@ STAGE PLANS: predicate: (key = 86) (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: '86' (type: string), val (type: string), foo (type: string) + expressions: '86' (type: string), val (type: string), 'bar' (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/input23.q.out b/ql/src/test/results/clientpositive/input23.q.out index fefcfa5..51a2329 100644 --- a/ql/src/test/results/clientpositive/input23.q.out +++ b/ql/src/test/results/clientpositive/input23.q.out @@ -73,7 +73,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE tag: 0 - value expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) + value expressions: key (type: string), value (type: string) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -132,12 +132,12 @@ STAGE PLANS: condition map: Inner Join 0 to 1 condition expressions: - 0 {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} + 0 {VALUE._col0} {VALUE._col1} 1 {VALUE._col0} {VALUE._col1} - outputColumnNames: _col0, _col1, _col2, _col3, _col7, _col8 + outputColumnNames: _col0, _col1, _col7, _col8 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col7 (type: string), _col8 (type: string), '2008-04-08' (type: string), '14' (type: string) + expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), '11' (type: string), _col7 (type: string), _col8 (type: string), '2008-04-08' (type: string), '14' (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Limit diff --git a/ql/src/test/results/clientpositive/input25.q.out b/ql/src/test/results/clientpositive/input25.q.out index 84e5f59..6d8cb15 100644 --- a/ql/src/test/results/clientpositive/input25.q.out +++ b/ql/src/test/results/clientpositive/input25.q.out @@ -48,8 +48,8 @@ STAGE PLANS: alias: x Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: a (type: int), b (type: int), d (type: string) - outputColumnNames: _col0, _col1, _col2 + expressions: a (type: int), b (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Limit Number of rows: 10 @@ -57,10 +57,10 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + value expressions: _col0 (type: int), _col1 (type: int) Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), VALUE._col2 (type: string) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), '2009-01-01' (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Limit @@ -112,8 +112,8 @@ STAGE PLANS: alias: x Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: a (type: int), b (type: int), d (type: string) - outputColumnNames: _col0, _col1, _col2 + expressions: a (type: int), b (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Limit Number of rows: 10 @@ -121,10 +121,10 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + value expressions: _col0 (type: int), _col1 (type: int) Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), VALUE._col2 (type: string) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), '2009-02-02' (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Limit diff --git a/ql/src/test/results/clientpositive/input26.q.out b/ql/src/test/results/clientpositive/input26.q.out index bd8cf9e..443c1c5 100644 --- a/ql/src/test/results/clientpositive/input26.q.out +++ b/ql/src/test/results/clientpositive/input26.q.out @@ -26,18 +26,18 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string) + value expressions: _col1 (type: string) Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), '11' (type: string) + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 5 @@ -56,7 +56,7 @@ STAGE PLANS: Union Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), _col3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -70,7 +70,7 @@ STAGE PLANS: Union Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), _col3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -85,8 +85,8 @@ STAGE PLANS: Map Reduce Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), '2008-04-08' (type: string), '14' (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), '14' (type: string) + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Limit Number of rows: 5 diff --git a/ql/src/test/results/clientpositive/input42.q.out b/ql/src/test/results/clientpositive/input42.q.out index 69999d3..8ecb492 100644 --- a/ql/src/test/results/clientpositive/input42.q.out +++ b/ql/src/test/results/clientpositive/input42.q.out @@ -134,7 +134,7 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE GatherStats: false Select Operator - expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) + expressions: key (type: string), value (type: string), '2008-04-08' (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE ListSink @@ -1203,7 +1203,7 @@ STAGE PLANS: predicate: (key < 200) (type: boolean) Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) + expressions: key (type: string), value (type: string), '2008-04-08' (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -1775,7 +1775,7 @@ STAGE PLANS: predicate: (rand(100) < 0.1) (type: boolean) Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) + expressions: key (type: string), value (type: string), '2008-04-08' (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/input_part0.q.out b/ql/src/test/results/clientpositive/input_part0.q.out index 0f928b5..f1853aa 100644 --- a/ql/src/test/results/clientpositive/input_part0.q.out +++ b/ql/src/test/results/clientpositive/input_part0.q.out @@ -16,7 +16,7 @@ STAGE PLANS: alias: x Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) + expressions: key (type: string), value (type: string), '2008-04-08' (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE ListSink diff --git a/ql/src/test/results/clientpositive/input_part1.q.out b/ql/src/test/results/clientpositive/input_part1.q.out index 1e094ba..64a57bb 100644 --- a/ql/src/test/results/clientpositive/input_part1.q.out +++ b/ql/src/test/results/clientpositive/input_part1.q.out @@ -93,7 +93,7 @@ STAGE PLANS: predicate: (key < 100) (type: boolean) Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), hr (type: string), ds (type: string) + expressions: UDFToInteger(key) (type: int), value (type: string), '12' (type: string), '2008-04-08' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/input_part2.q.out b/ql/src/test/results/clientpositive/input_part2.q.out index 2dcd682..e0b32b1 100644 --- a/ql/src/test/results/clientpositive/input_part2.q.out +++ b/ql/src/test/results/clientpositive/input_part2.q.out @@ -157,7 +157,7 @@ STAGE PLANS: predicate: ((key < 100) and (ds = '2008-04-08')) (type: boolean) Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), hr (type: string), '2008-04-08' (type: string) + expressions: UDFToInteger(key) (type: int), value (type: string), '12' (type: string), '2008-04-08' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -191,7 +191,7 @@ STAGE PLANS: predicate: ((key < 100) and (ds = '2008-04-09')) (type: boolean) Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), hr (type: string), '2008-04-09' (type: string) + expressions: UDFToInteger(key) (type: int), value (type: string), '12' (type: string), '2008-04-09' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/input_part3.q.out b/ql/src/test/results/clientpositive/input_part3.q.out index 380e019..3514bb7 100644 --- a/ql/src/test/results/clientpositive/input_part3.q.out +++ b/ql/src/test/results/clientpositive/input_part3.q.out @@ -16,7 +16,7 @@ STAGE PLANS: alias: x Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) + expressions: key (type: string), value (type: string), '2008-04-08' (type: string), '11' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE ListSink diff --git a/ql/src/test/results/clientpositive/input_part5.q.out b/ql/src/test/results/clientpositive/input_part5.q.out index 6077ec8..ffc9251 100644 --- a/ql/src/test/results/clientpositive/input_part5.q.out +++ b/ql/src/test/results/clientpositive/input_part5.q.out @@ -35,7 +35,7 @@ STAGE PLANS: predicate: (key < 100) (type: boolean) Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) + expressions: key (type: string), value (type: string), '2008-04-08' (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/input_part7.q.out b/ql/src/test/results/clientpositive/input_part7.q.out index a9c003b..13c7857 100644 --- a/ql/src/test/results/clientpositive/input_part7.q.out +++ b/ql/src/test/results/clientpositive/input_part7.q.out @@ -126,17 +126,17 @@ STAGE PLANS: predicate: (key < 100) (type: boolean) Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + expressions: key (type: string), value (type: string), hr (type: string) + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE Union Statistics: Num rows: 666 Data size: 7074 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string) + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 666 Data size: 7074 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + key expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), _col3 (type: string) sort order: ++++ Statistics: Num rows: 666 Data size: 7074 Basic stats: COMPLETE Column stats: NONE tag: -1 @@ -150,17 +150,17 @@ STAGE PLANS: predicate: (key < 100) (type: boolean) Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + expressions: key (type: string), value (type: string), hr (type: string) + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE Union Statistics: Num rows: 666 Data size: 7074 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string) + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 666 Data size: 7074 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + key expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), _col3 (type: string) sort order: ++++ Statistics: Num rows: 666 Data size: 7074 Basic stats: COMPLETE Column stats: NONE tag: -1 @@ -266,7 +266,7 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string) + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), '2008-04-08' (type: string), KEY.reducesinkkey3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 666 Data size: 7074 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/input_part8.q.out b/ql/src/test/results/clientpositive/input_part8.q.out index 0895354..0fef273 100644 --- a/ql/src/test/results/clientpositive/input_part8.q.out +++ b/ql/src/test/results/clientpositive/input_part8.q.out @@ -16,7 +16,7 @@ STAGE PLANS: alias: x Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) + expressions: key (type: string), value (type: string), '2008-04-08' (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Limit diff --git a/ql/src/test/results/clientpositive/input_part9.q.out b/ql/src/test/results/clientpositive/input_part9.q.out index bfbdc5f..8586a67 100644 --- a/ql/src/test/results/clientpositive/input_part9.q.out +++ b/ql/src/test/results/clientpositive/input_part9.q.out @@ -54,7 +54,7 @@ STAGE PLANS: predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) + expressions: key (type: string), value (type: string), '2008-04-08' (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/insert0.q.out b/ql/src/test/results/clientpositive/insert0.q.out new file mode 100644 index 0000000..e83bae1 --- /dev/null +++ b/ql/src/test/results/clientpositive/insert0.q.out @@ -0,0 +1,208 @@ +PREHOOK: query: DROP TABLE insert_into1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE insert_into1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE ctas_table +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE ctas_table +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE ctas_part +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE ctas_part +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE insert_into1 (key int, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@insert_into1 +POSTHOOK: query: CREATE TABLE insert_into1 (key int, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@insert_into1 +PREHOOK: query: INSERT OVERWRITE TABLE insert_into1 SELECT * from src ORDER BY key LIMIT 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@insert_into1 +POSTHOOK: query: INSERT OVERWRITE TABLE insert_into1 SELECT * from src ORDER BY key LIMIT 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@insert_into1 +POSTHOOK: Lineage: insert_into1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: insert_into1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from insert_into1 order by key +PREHOOK: type: QUERY +PREHOOK: Input: default@insert_into1 +#### A masked pattern was here #### +POSTHOOK: query: select * from insert_into1 order by key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@insert_into1 +#### A masked pattern was here #### +0 val_0 +0 val_0 +0 val_0 +10 val_10 +100 val_100 +100 val_100 +103 val_103 +103 val_103 +104 val_104 +104 val_104 +PREHOOK: query: INSERT INTO TABLE insert_into1 SELECT * from src ORDER BY key DESC LIMIT 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@insert_into1 +POSTHOOK: query: INSERT INTO TABLE insert_into1 SELECT * from src ORDER BY key DESC LIMIT 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@insert_into1 +POSTHOOK: Lineage: insert_into1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: insert_into1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from insert_into1 order by key +PREHOOK: type: QUERY +PREHOOK: Input: default@insert_into1 +#### A masked pattern was here #### +POSTHOOK: query: select * from insert_into1 order by key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@insert_into1 +#### A masked pattern was here #### +0 val_0 +0 val_0 +0 val_0 +10 val_10 +90 val_90 +90 val_90 +92 val_92 +95 val_95 +95 val_95 +96 val_96 +97 val_97 +97 val_97 +98 val_98 +98 val_98 +100 val_100 +100 val_100 +103 val_103 +103 val_103 +104 val_104 +104 val_104 +PREHOOK: query: create table ctas_table as SELECT key, count(value) as foo from src GROUP BY key LIMIT 10 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@ctas_table +POSTHOOK: query: create table ctas_table as SELECT key, count(value) as foo from src GROUP BY key LIMIT 10 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ctas_table +PREHOOK: query: describe extended ctas_table +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ctas_table +POSTHOOK: query: describe extended ctas_table +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ctas_table +key string +foo bigint + +#### A masked pattern was here #### +PREHOOK: query: select * from ctas_table order by key +PREHOOK: type: QUERY +PREHOOK: Input: default@ctas_table +#### A masked pattern was here #### +POSTHOOK: query: select * from ctas_table order by key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ctas_table +#### A masked pattern was here #### +0 3 +10 1 +100 2 +103 2 +104 2 +105 1 +11 1 +111 1 +113 2 +114 1 +PREHOOK: query: create table ctas_part (key int, value string) partitioned by (modkey bigint) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ctas_part +POSTHOOK: query: create table ctas_part (key int, value string) partitioned by (modkey bigint) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ctas_part +PREHOOK: query: insert overwrite table ctas_part partition (modkey) +select key, value, ceil(key / 100) from src where key is not null limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@ctas_part +POSTHOOK: query: insert overwrite table ctas_part partition (modkey) +select key, value, ceil(key / 100) from src where key is not null limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@ctas_part@modkey=1 +POSTHOOK: Output: default@ctas_part@modkey=2 +POSTHOOK: Output: default@ctas_part@modkey=3 +POSTHOOK: Output: default@ctas_part@modkey=4 +POSTHOOK: Output: default@ctas_part@modkey=5 +POSTHOOK: Lineage: ctas_part PARTITION(modkey=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_part PARTITION(modkey=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_part PARTITION(modkey=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_part PARTITION(modkey=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_part PARTITION(modkey=3).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_part PARTITION(modkey=3).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_part PARTITION(modkey=4).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_part PARTITION(modkey=4).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_part PARTITION(modkey=5).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ctas_part PARTITION(modkey=5).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from ctas_part order by key +PREHOOK: type: QUERY +PREHOOK: Input: default@ctas_part +PREHOOK: Input: default@ctas_part@modkey=1 +PREHOOK: Input: default@ctas_part@modkey=2 +PREHOOK: Input: default@ctas_part@modkey=3 +PREHOOK: Input: default@ctas_part@modkey=4 +PREHOOK: Input: default@ctas_part@modkey=5 +#### A masked pattern was here #### +POSTHOOK: query: select * from ctas_part order by key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ctas_part +POSTHOOK: Input: default@ctas_part@modkey=1 +POSTHOOK: Input: default@ctas_part@modkey=2 +POSTHOOK: Input: default@ctas_part@modkey=3 +POSTHOOK: Input: default@ctas_part@modkey=4 +POSTHOOK: Input: default@ctas_part@modkey=5 +#### A masked pattern was here #### +27 val_27 1 +86 val_86 1 +98 val_98 1 +165 val_165 2 +238 val_238 3 +255 val_255 3 +278 val_278 3 +311 val_311 4 +409 val_409 5 +484 val_484 5 +PREHOOK: query: DROP TABLE insert_into1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@insert_into1 +PREHOOK: Output: default@insert_into1 +POSTHOOK: query: DROP TABLE insert_into1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@insert_into1 +POSTHOOK: Output: default@insert_into1 +PREHOOK: query: DROP TABLE ctas_table +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ctas_table +PREHOOK: Output: default@ctas_table +POSTHOOK: query: DROP TABLE ctas_table +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ctas_table +POSTHOOK: Output: default@ctas_table +PREHOOK: query: DROP TABLE ctas_part +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ctas_part +PREHOOK: Output: default@ctas_part +POSTHOOK: query: DROP TABLE ctas_part +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ctas_part +POSTHOOK: Output: default@ctas_part diff --git a/ql/src/test/results/clientpositive/join11.q.out b/ql/src/test/results/clientpositive/join11.q.out index 3c8e742..9e77bb7 100644 --- a/ql/src/test/results/clientpositive/join11.q.out +++ b/ql/src/test/results/clientpositive/join11.q.out @@ -26,33 +26,33 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 100) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 100) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 100) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 100) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -61,14 +61,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {VALUE._col0} outputColumnNames: _col0, _col3 - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/join12.q.out b/ql/src/test/results/clientpositive/join12.q.out index 860565e..ee4b6ac 100644 --- a/ql/src/test/results/clientpositive/join12.q.out +++ b/ql/src/test/results/clientpositive/join12.q.out @@ -32,48 +32,48 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key < 100) and key is not null) and (key < 80)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: (((key < 100) and (key < 80)) and key is not null) (type: boolean) + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key < 80) and key is not null) and (key < 100)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 80) and (key < 100)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key < 100) and key is not null) and (key < 80)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 100) and (key < 80)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -84,14 +84,14 @@ STAGE PLANS: 1 {VALUE._col0} 2 outputColumnNames: _col0, _col3 - Statistics: Num rows: 59 Data size: 629 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 59 Data size: 629 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 59 Data size: 629 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/join13.q.out b/ql/src/test/results/clientpositive/join13.q.out index 753e292..f5f3e08 100644 --- a/ql/src/test/results/clientpositive/join13.q.out +++ b/ql/src/test/results/clientpositive/join13.q.out @@ -33,33 +33,33 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 100) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 100) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 100) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 100) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -68,10 +68,10 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col2, _col3 - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_col0 + _col2) is not null (type: boolean) - Statistics: Num rows: 46 Data size: 489 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -87,7 +87,7 @@ STAGE PLANS: key expressions: (_col0 + _col2) (type: double) sort order: + Map-reduce partition columns: (_col0 + _col2) (type: double) - Statistics: Num rows: 46 Data size: 489 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col3 (type: string) TableScan alias: src @@ -112,14 +112,14 @@ STAGE PLANS: 0 {VALUE._col0} {VALUE._col3} 1 outputColumnNames: _col0, _col3 - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 100 Data size: 1065 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 100 Data size: 1065 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 100 Data size: 1065 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/join14.q.out b/ql/src/test/results/clientpositive/join14.q.out index 4d4f3e0..f30345b 100644 --- a/ql/src/test/results/clientpositive/join14.q.out +++ b/ql/src/test/results/clientpositive/join14.q.out @@ -43,13 +43,13 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > 100) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key > 100) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: diff --git a/ql/src/test/results/clientpositive/join16.q.out b/ql/src/test/results/clientpositive/join16.q.out index 8463837..4573cb6 100644 --- a/ql/src/test/results/clientpositive/join16.q.out +++ b/ql/src/test/results/clientpositive/join16.q.out @@ -14,22 +14,22 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((key > 10) and (key > 20)) and key is not null) and value is not null) and (value < 200)) (type: boolean) - Statistics: Num rows: 4 Data size: 42 Basic stats: COMPLETE Column stats: NONE + predicate: ((((key > 10) and value is not null) and (key > 20)) and (value < 200)) (type: boolean) + Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 4 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE TableScan alias: tab Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((key > 20) and key is not null) and value is not null) and (value < 200)) (type: boolean) + predicate: ((((key > 20) and value is not null) and key is not null) and (value < 200)) (type: boolean) Statistics: Num rows: 14 Data size: 148 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string), value (type: string) diff --git a/ql/src/test/results/clientpositive/join34.q.out b/ql/src/test/results/clientpositive/join34.q.out index 20038d7..36aad35 100644 --- a/ql/src/test/results/clientpositive/join34.q.out +++ b/ql/src/test/results/clientpositive/join34.q.out @@ -179,14 +179,14 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: ((key < 20) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 20) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 332 Data size: 3526 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -198,17 +198,17 @@ STAGE PLANS: 1 key (type: string) outputColumnNames: _col1, _col2, _col3 Position of Big Table: 0 - Statistics: Num rows: 182 Data size: 1938 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 365 Data size: 3878 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col3 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 182 Data size: 1938 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 365 Data size: 3878 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 182 Data size: 1938 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 365 Data size: 3878 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -235,14 +235,14 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: ((key > 100) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key > 100) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 332 Data size: 3526 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -254,17 +254,17 @@ STAGE PLANS: 1 key (type: string) outputColumnNames: _col1, _col2, _col3 Position of Big Table: 0 - Statistics: Num rows: 182 Data size: 1938 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 365 Data size: 3878 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col3 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 182 Data size: 1938 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 365 Data size: 3878 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 182 Data size: 1938 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 365 Data size: 3878 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git a/ql/src/test/results/clientpositive/join35.q.out b/ql/src/test/results/clientpositive/join35.q.out index 1822f2e..70fcf8f 100644 --- a/ql/src/test/results/clientpositive/join35.q.out +++ b/ql/src/test/results/clientpositive/join35.q.out @@ -164,23 +164,23 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: ((key < 20) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 20) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: key - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: bigint) auto parallelism: false @@ -240,11 +240,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 @@ -294,7 +294,7 @@ STAGE PLANS: TableScan GatherStats: false Union - Statistics: Num rows: 82 Data size: 870 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -306,17 +306,17 @@ STAGE PLANS: 1 key (type: string) outputColumnNames: _col1, _col2, _col3 Position of Big Table: 0 - Statistics: Num rows: 90 Data size: 957 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1938 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col3 (type: string), UDFToInteger(_col1) (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 90 Data size: 957 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1938 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 90 Data size: 957 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1938 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -340,7 +340,7 @@ STAGE PLANS: TableScan GatherStats: false Union - Statistics: Num rows: 82 Data size: 870 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -352,17 +352,17 @@ STAGE PLANS: 1 key (type: string) outputColumnNames: _col1, _col2, _col3 Position of Big Table: 0 - Statistics: Num rows: 90 Data size: 957 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1938 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col3 (type: string), UDFToInteger(_col1) (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 90 Data size: 957 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1938 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 90 Data size: 957 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1938 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -510,23 +510,23 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: ((key > 100) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key > 100) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: key - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: bigint) auto parallelism: false @@ -586,11 +586,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 diff --git a/ql/src/test/results/clientpositive/join38.q.out b/ql/src/test/results/clientpositive/join38.q.out index a8a0603..d3acf11 100644 --- a/ql/src/test/results/clientpositive/join38.q.out +++ b/ql/src/test/results/clientpositive/join38.q.out @@ -66,8 +66,8 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key = 111)) (type: boolean) - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 111) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: 0 {value} @@ -83,8 +83,8 @@ STAGE PLANS: alias: b Statistics: Num rows: 2 Data size: 126 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (col11 is not null and (col11 = 111)) (type: boolean) - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + predicate: (col11 = 111) (type: boolean) + Statistics: Num rows: 1 Data size: 63 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -95,22 +95,22 @@ STAGE PLANS: 0 '111' (type: string) 1 '111' (type: string) outputColumnNames: _col1, _col10 - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col10 (type: string) outputColumnNames: _col1, _col10 - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col1 (type: string), _col10 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Local Work: Map Reduce Local Work @@ -120,14 +120,14 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 68 Data size: 724 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 68 Data size: 724 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 68 Data size: 724 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/join_vc.q.out b/ql/src/test/results/clientpositive/join_vc.q.out index 12004ca..45652ff 100644 --- a/ql/src/test/results/clientpositive/join_vc.q.out +++ b/ql/src/test/results/clientpositive/join_vc.q.out @@ -137,3 +137,227 @@ POSTHOOK: Input: default@src 0 238 val_238 0 238 val_238 0 238 val_238 +PREHOOK: query: explain +select t2.BLOCK__OFFSET__INSIDE__FILE +from src t1 join src t2 on t1.key = t2.key where t1.key < 100 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select t2.BLOCK__OFFSET__INSIDE__FILE +from src t1 join src t2 on t1.key = t2.key where t1.key < 100 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key < 100) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + value expressions: BLOCK__OFFSET__INSIDE__FILE (type: bigint) + TableScan + alias: t1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key < 100) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {VALUE._col1} + outputColumnNames: _col7 + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col7 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select t2.BLOCK__OFFSET__INSIDE__FILE +from src t1 join src t2 on t1.key = t2.key where t1.key < 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select t2.BLOCK__OFFSET__INSIDE__FILE +from src t1 join src t2 on t1.key = t2.key where t1.key < 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +2088 +2632 +968 +2088 +2632 +968 +2088 +2632 +968 +2846 +3170 +1720 +4362 +1720 +4362 +386 +2770 +386 +2770 +910 +5340 +5514 +5340 +5514 +2824 +4004 +1118 +4594 +1972 +4594 +1972 +2226 +5284 +2226 +5284 +34 +5616 +3494 +3592 +3192 +3138 +4012 +1238 +3138 +4012 +1238 +3138 +4012 +1238 +5626 +328 +5626 +328 +1218 +3388 +2030 +3298 +2030 +3298 +2330 +4068 +1198 +3060 +4540 +3864 +3060 +4540 +3864 +3060 +4540 +3864 +2308 +1462 +2308 +1462 +4186 +1440 +1024 +1906 +3128 +1906 +3128 +3516 +1592 +198 +1754 +5306 +1754 +5306 +3570 +3794 +4640 +4548 +3794 +4640 +4548 +3794 +4640 +4548 +2792 +1208 +2792 +1208 +3548 +3378 +3538 +3378 +3538 +2622 +3368 +1916 +4058 +396 +5070 +1674 +5070 +1674 +1872 +5606 +1872 +5606 +2612 +12 +2652 +5398 +2802 +5744 +4304 +2802 +5744 +4304 +2802 +5744 +4304 +1176 +3160 +2400 +3160 +2400 +2216 +5572 +5802 +5572 +5802 +92 +2458 +92 +2458 diff --git a/ql/src/test/results/clientpositive/lateral_view.q.out b/ql/src/test/results/clientpositive/lateral_view.q.out index 8b87c37..66c2968 100644 --- a/ql/src/test/results/clientpositive/lateral_view.q.out +++ b/ql/src/test/results/clientpositive/lateral_view.q.out @@ -142,10 +142,10 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 28000 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 3 - Statistics: Num rows: 3 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -166,10 +166,10 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 28000 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 3 - Statistics: Num rows: 3 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -208,10 +208,10 @@ STAGE PLANS: Select Operator expressions: _col5 (type: int) outputColumnNames: _col5 - Statistics: Num rows: 1000 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 268000 Basic stats: COMPLETE Column stats: COMPLETE Lateral View Join Operator outputColumnNames: _col5, _col6 - Statistics: Num rows: 2000 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 268000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col5 (type: int), _col6 (type: string) outputColumnNames: _col0, _col1 @@ -235,7 +235,7 @@ STAGE PLANS: function name: explode Lateral View Join Operator outputColumnNames: _col5, _col6 - Statistics: Num rows: 2000 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 268000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col5 (type: int), _col6 (type: string) outputColumnNames: _col0, _col1 @@ -265,10 +265,10 @@ STAGE PLANS: Select Operator expressions: _col5 (type: int) outputColumnNames: _col5 - Statistics: Num rows: 1000 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 268000 Basic stats: COMPLETE Column stats: COMPLETE Lateral View Join Operator outputColumnNames: _col5, _col6 - Statistics: Num rows: 2000 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 268000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col5 (type: int), _col6 (type: string) outputColumnNames: _col0, _col1 @@ -292,7 +292,7 @@ STAGE PLANS: function name: explode Lateral View Join Operator outputColumnNames: _col5, _col6 - Statistics: Num rows: 2000 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 268000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col5 (type: int), _col6 (type: string) outputColumnNames: _col0, _col1 @@ -339,14 +339,14 @@ STAGE PLANS: Lateral View Forward Statistics: Num rows: 1000 Data size: 24000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - Statistics: Num rows: 1000 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 268000 Basic stats: COMPLETE Column stats: COMPLETE Lateral View Join Operator outputColumnNames: _col6 - Statistics: Num rows: 2000 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 268000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col6 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 268000 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 3 Statistics: Num rows: 3 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE @@ -366,11 +366,11 @@ STAGE PLANS: function name: explode Lateral View Join Operator outputColumnNames: _col6 - Statistics: Num rows: 2000 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 268000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col6 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 268000 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 3 Statistics: Num rows: 3 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE @@ -394,14 +394,14 @@ STAGE PLANS: Lateral View Forward Statistics: Num rows: 1000 Data size: 24000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - Statistics: Num rows: 1000 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 268000 Basic stats: COMPLETE Column stats: COMPLETE Lateral View Join Operator outputColumnNames: _col6 - Statistics: Num rows: 2000 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 268000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col6 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 268000 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 3 Statistics: Num rows: 3 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE @@ -421,11 +421,11 @@ STAGE PLANS: function name: explode Lateral View Join Operator outputColumnNames: _col6 - Statistics: Num rows: 2000 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 268000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col6 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 268000 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 3 Statistics: Num rows: 3 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE diff --git a/ql/src/test/results/clientpositive/lateral_view_noalias.q.out b/ql/src/test/results/clientpositive/lateral_view_noalias.q.out index 9b6e9c4..e1445bf 100644 --- a/ql/src/test/results/clientpositive/lateral_view_noalias.q.out +++ b/ql/src/test/results/clientpositive/lateral_view_noalias.q.out @@ -28,10 +28,10 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 192000 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 2 - Statistics: Num rows: 2 Data size: 768 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 768 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -52,10 +52,10 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 192000 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 2 - Statistics: Num rows: 2 Data size: 768 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 768 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/lateral_view_ppd.q.out b/ql/src/test/results/clientpositive/lateral_view_ppd.q.out index b186192..1220030 100644 --- a/ql/src/test/results/clientpositive/lateral_view_ppd.q.out +++ b/ql/src/test/results/clientpositive/lateral_view_ppd.q.out @@ -175,44 +175,23 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Lateral View Forward - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: value - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Lateral View Join Operator - outputColumnNames: _col1, _col7 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: string), _col7 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 12 - Statistics: Num rows: 12 Data size: 120 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 12 Data size: 120 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Select Operator - expressions: array(1,2,3) (type: array) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - UDTF Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - function name: explode + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((ds = '2008-04-08') and (hr = '12')) (type: boolean) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Lateral View Forward + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Lateral View Join Operator outputColumnNames: _col1, _col7 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4000 Data size: 42496 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col7 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4000 Data size: 42496 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 12 Statistics: Num rows: 12 Data size: 120 Basic stats: COMPLETE Column stats: NONE @@ -223,6 +202,30 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: array(1,2,3) (type: array) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + UDTF Operator + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + function name: explode + Lateral View Join Operator + outputColumnNames: _col1, _col7 + Statistics: Num rows: 4000 Data size: 42496 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: string), _col7 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4000 Data size: 42496 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 12 + Statistics: Num rows: 12 Data size: 120 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 120 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -233,12 +236,18 @@ STAGE PLANS: PREHOOK: query: SELECT value, myCol FROM (SELECT * FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol) a WHERE ds='2008-04-08' AND hr="12" LIMIT 12 PREHOOK: type: QUERY PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### POSTHOOK: query: SELECT value, myCol FROM (SELECT * FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol) a WHERE ds='2008-04-08' AND hr="12" LIMIT 12 POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### val_238 1 val_238 2 diff --git a/ql/src/test/results/clientpositive/limit_partition_metadataonly.q.out b/ql/src/test/results/clientpositive/limit_partition_metadataonly.q.out index 354d440..7513886 100644 --- a/ql/src/test/results/clientpositive/limit_partition_metadataonly.q.out +++ b/ql/src/test/results/clientpositive/limit_partition_metadataonly.q.out @@ -14,7 +14,7 @@ STAGE PLANS: alias: srcpart Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: ds (type: string) + expressions: '2008-04-08' (type: string) outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE ListSink diff --git a/ql/src/test/results/clientpositive/list_bucket_dml_12.q.out b/ql/src/test/results/clientpositive/list_bucket_dml_12.q.out index b041d52..ea4b6b8 100644 --- a/ql/src/test/results/clientpositive/list_bucket_dml_12.q.out +++ b/ql/src/test/results/clientpositive/list_bucket_dml_12.q.out @@ -332,7 +332,7 @@ STAGE PLANS: predicate: ((col2 = '466') and (col4 = 'val_466')) (type: boolean) Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: col1 (type: string), '466' (type: string), col3 (type: string), 'val_466' (type: string), col5 (type: string), ds (type: string), hr (type: string) + expressions: col1 (type: string), '466' (type: string), col3 (type: string), 'val_466' (type: string), col5 (type: string), '2008-04-08' (type: string), '11' (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -490,7 +490,7 @@ STAGE PLANS: predicate: ((col2 = '382') and (col4 = 'val_382')) (type: boolean) Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: col1 (type: string), '382' (type: string), col3 (type: string), 'val_382' (type: string), col5 (type: string), ds (type: string), hr (type: string) + expressions: col1 (type: string), '382' (type: string), col3 (type: string), 'val_382' (type: string), col5 (type: string), '2008-04-08' (type: string), '11' (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/list_bucket_dml_13.q.out b/ql/src/test/results/clientpositive/list_bucket_dml_13.q.out index afe201d..9b30a84 100644 --- a/ql/src/test/results/clientpositive/list_bucket_dml_13.q.out +++ b/ql/src/test/results/clientpositive/list_bucket_dml_13.q.out @@ -332,7 +332,7 @@ STAGE PLANS: predicate: ((col2 = '466') and (col4 = 'val_466')) (type: boolean) Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: col1 (type: string), '466' (type: string), col3 (type: string), 'val_466' (type: string), col5 (type: string), ds (type: string), hr (type: string) + expressions: col1 (type: string), '466' (type: string), col3 (type: string), 'val_466' (type: string), col5 (type: string), '2008-04-08' (type: string), '2013-01-23+18:00:99' (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/list_bucket_dml_2.q.out b/ql/src/test/results/clientpositive/list_bucket_dml_2.q.out index 34e5995..01f7218 100644 --- a/ql/src/test/results/clientpositive/list_bucket_dml_2.q.out +++ b/ql/src/test/results/clientpositive/list_bucket_dml_2.q.out @@ -418,7 +418,7 @@ STAGE PLANS: predicate: ((key = '484') and (value = 'val_484')) (type: boolean) Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: '484' (type: string), 'val_484' (type: string), ds (type: string), hr (type: string) + expressions: '484' (type: string), 'val_484' (type: string), '2008-04-08' (type: string), '11' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/list_bucket_dml_4.q.out b/ql/src/test/results/clientpositive/list_bucket_dml_4.q.out index 671653e..b517c83 100644 --- a/ql/src/test/results/clientpositive/list_bucket_dml_4.q.out +++ b/ql/src/test/results/clientpositive/list_bucket_dml_4.q.out @@ -829,7 +829,7 @@ STAGE PLANS: predicate: ((key = '484') and (value = 'val_484')) (type: boolean) Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: '484' (type: string), 'val_484' (type: string), ds (type: string), hr (type: string) + expressions: '484' (type: string), 'val_484' (type: string), '2008-04-08' (type: string), '11' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/list_bucket_dml_5.q.out b/ql/src/test/results/clientpositive/list_bucket_dml_5.q.out index 0610fa2..d8a0abf 100644 --- a/ql/src/test/results/clientpositive/list_bucket_dml_5.q.out +++ b/ql/src/test/results/clientpositive/list_bucket_dml_5.q.out @@ -462,7 +462,7 @@ STAGE PLANS: predicate: ((key = '103') and (value = 'val_103')) (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: '103' (type: string), 'val_103' (type: string), ds (type: string), hr (type: string) + expressions: '103' (type: string), 'val_103' (type: string), '2008-04-08' (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/list_bucket_dml_9.q.out b/ql/src/test/results/clientpositive/list_bucket_dml_9.q.out index 4ac64b1..135cd78 100644 --- a/ql/src/test/results/clientpositive/list_bucket_dml_9.q.out +++ b/ql/src/test/results/clientpositive/list_bucket_dml_9.q.out @@ -829,7 +829,7 @@ STAGE PLANS: predicate: ((key = '484') and (value = 'val_484')) (type: boolean) Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: '484' (type: string), 'val_484' (type: string), ds (type: string), hr (type: string) + expressions: '484' (type: string), 'val_484' (type: string), '2008-04-08' (type: string), '11' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/list_bucket_query_multiskew_3.q.out b/ql/src/test/results/clientpositive/list_bucket_query_multiskew_3.q.out index 0e47417..d738dbf 100644 --- a/ql/src/test/results/clientpositive/list_bucket_query_multiskew_3.q.out +++ b/ql/src/test/results/clientpositive/list_bucket_query_multiskew_3.q.out @@ -288,7 +288,7 @@ STAGE PLANS: predicate: (key = '145') (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: '145' (type: string), value (type: string), ds (type: string), hr (type: string) + expressions: '145' (type: string), value (type: string), '1' (type: string), '1' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -601,7 +601,7 @@ STAGE PLANS: predicate: ((key = '484') and (value = 'val_484')) (type: boolean) Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: '484' (type: string), 'val_484' (type: string), ds (type: string), hr (type: string) + expressions: '484' (type: string), 'val_484' (type: string), '1' (type: string), '2' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -755,7 +755,7 @@ STAGE PLANS: predicate: ((key = '327') and (value = 'val_327')) (type: boolean) Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: '327' (type: string), 'val_327' (type: string), ds (type: string), hr (type: string) + expressions: '327' (type: string), 'val_327' (type: string), '1' (type: string), '3' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/list_bucket_query_oneskew_2.q.out b/ql/src/test/results/clientpositive/list_bucket_query_oneskew_2.q.out index 85fc092..0ef749a 100644 --- a/ql/src/test/results/clientpositive/list_bucket_query_oneskew_2.q.out +++ b/ql/src/test/results/clientpositive/list_bucket_query_oneskew_2.q.out @@ -836,7 +836,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: bigint) + expressions: 484 (type: int), _col1 (type: bigint) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/mapjoin_mapjoin.q.out b/ql/src/test/results/clientpositive/mapjoin_mapjoin.q.out index 79f334c..7da7122 100644 --- a/ql/src/test/results/clientpositive/mapjoin_mapjoin.q.out +++ b/ql/src/test/results/clientpositive/mapjoin_mapjoin.q.out @@ -478,8 +478,8 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (value is not null and (value > 'val_450')) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (value > 'val_450') (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: 0 {key} diff --git a/ql/src/test/results/clientpositive/merge_dynamic_partition.q.out b/ql/src/test/results/clientpositive/merge_dynamic_partition.q.out index 85294bf..da19b32 100644 --- a/ql/src/test/results/clientpositive/merge_dynamic_partition.q.out +++ b/ql/src/test/results/clientpositive/merge_dynamic_partition.q.out @@ -1291,7 +1291,7 @@ STAGE PLANS: alias: srcpart_merge_dp Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) + expressions: key (type: string), value (type: string), '2008-04-08' (type: string), '11' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/nonmr_fetch.q.out b/ql/src/test/results/clientpositive/nonmr_fetch.q.out index 87502cf..c6a4318 100644 --- a/ql/src/test/results/clientpositive/nonmr_fetch.q.out +++ b/ql/src/test/results/clientpositive/nonmr_fetch.q.out @@ -58,7 +58,7 @@ STAGE PLANS: alias: srcpart Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) + expressions: key (type: string), value (type: string), '2008-04-08' (type: string), '11' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Limit @@ -313,7 +313,7 @@ STAGE PLANS: alias: srcpart Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) + expressions: key (type: string), value (type: string), '2008-04-08' (type: string), '11' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Limit diff --git a/ql/src/test/results/clientpositive/nonmr_fetch_threshold.q.out b/ql/src/test/results/clientpositive/nonmr_fetch_threshold.q.out index 1749aa4..cb0d332 100644 --- a/ql/src/test/results/clientpositive/nonmr_fetch_threshold.q.out +++ b/ql/src/test/results/clientpositive/nonmr_fetch_threshold.q.out @@ -14,7 +14,7 @@ STAGE PLANS: alias: srcpart Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) + expressions: key (type: string), value (type: string), '2008-04-08' (type: string), '11' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Limit @@ -64,7 +64,7 @@ STAGE PLANS: alias: srcpart Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) + expressions: key (type: string), value (type: string), '2008-04-08' (type: string), '11' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Limit diff --git a/ql/src/test/results/clientpositive/optimize_nullscan.q.out b/ql/src/test/results/clientpositive/optimize_nullscan.q.out index fd77ad2..4e3640e 100644 --- a/ql/src/test/results/clientpositive/optimize_nullscan.q.out +++ b/ql/src/test/results/clientpositive/optimize_nullscan.q.out @@ -1772,34 +1772,27 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: srcpart - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - tag: 1 - auto parallelism: false - TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - tag: 0 - auto parallelism: false + Filter Operator + isSamplingPred: false + predicate: false (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + tag: 0 + auto parallelism: false Path -> Alias: -#### A masked pattern was here #### + -mr-10004default.src{} [null-subquery2:a-subquery2:src] Path -> Partition: -#### A masked pattern was here #### + -mr-10004default.src{} Partition base file name: src - input format: org.apache.hadoop.mapred.TextInputFormat + input format: org.apache.hadoop.hive.ql.io.OneNullRowInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: COLUMN_STATS_ACCURATE true @@ -1814,10 +1807,10 @@ STAGE PLANS: rawDataSize 5312 serialization.ddl struct src { string key, string value} serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serialization.lib org.apache.hadoop.hive.serde2.NullStructSerDe totalSize 5812 #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.NullStructSerDe input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1840,196 +1833,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src name: default.src -#### A masked pattern was here #### - Partition - base file name: hr=11 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - hr 11 - properties: - COLUMN_STATS_ACCURATE true - bucket_count -1 - columns key,value - columns.comments defaultdefault - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - numFiles 1 - numRows 500 - partition_columns ds/hr - partition_columns.types string:string - rawDataSize 5312 - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns key,value - columns.comments defaultdefault - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - partition_columns ds/hr - partition_columns.types string:string - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcpart - name: default.srcpart -#### A masked pattern was here #### - Partition - base file name: hr=12 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - hr 12 - properties: - COLUMN_STATS_ACCURATE true - bucket_count -1 - columns key,value - columns.comments defaultdefault - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - numFiles 1 - numRows 500 - partition_columns ds/hr - partition_columns.types string:string - rawDataSize 5312 - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns key,value - columns.comments defaultdefault - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - partition_columns ds/hr - partition_columns.types string:string - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcpart - name: default.srcpart -#### A masked pattern was here #### - Partition - base file name: hr=11 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-09 - hr 11 - properties: - COLUMN_STATS_ACCURATE true - bucket_count -1 - columns key,value - columns.comments defaultdefault - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - numFiles 1 - numRows 500 - partition_columns ds/hr - partition_columns.types string:string - rawDataSize 5312 - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns key,value - columns.comments defaultdefault - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - partition_columns ds/hr - partition_columns.types string:string - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcpart - name: default.srcpart -#### A masked pattern was here #### - Partition - base file name: hr=12 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-09 - hr 12 - properties: - COLUMN_STATS_ACCURATE true - bucket_count -1 - columns key,value - columns.comments defaultdefault - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - numFiles 1 - numRows 500 - partition_columns ds/hr - partition_columns.types string:string - rawDataSize 5312 - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns key,value - columns.comments defaultdefault - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - partition_columns ds/hr - partition_columns.types string:string - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcpart - name: default.srcpart Truncated Path -> Alias: - /src [null-subquery2:a-subquery2:src] - /srcpart/ds=2008-04-08/hr=11 [null-subquery2:a-subquery2:srcpart] - /srcpart/ds=2008-04-08/hr=12 [null-subquery2:a-subquery2:srcpart] - /srcpart/ds=2008-04-09/hr=11 [null-subquery2:a-subquery2:srcpart] - /srcpart/ds=2008-04-09/hr=12 [null-subquery2:a-subquery2:srcpart] + -mr-10004default.src{} [null-subquery2:a-subquery2:src] Needs Tagging: true Reduce Operator Tree: Join Operator @@ -2039,32 +1844,28 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE - Filter Operator - isSamplingPred: false - predicate: false (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types string - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types string + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Stage: Stage-2 Map Reduce @@ -2219,17 +2020,188 @@ PREHOOK: query: select * from (select key from src union all select src.key from PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Input: default@srcpart -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### POSTHOOK: query: select * from (select key from src union all select src.key from src left outer join srcpart on src.key = srcpart.key) a where false POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@srcpart -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +#### A masked pattern was here #### +PREHOOK: query: explain extended +select * from src s1, src s2 where false and s1.value = s2.value +PREHOOK: type: QUERY +POSTHOOK: query: explain extended +select * from src s1, src s2 where false and s1.value = s2.value +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_JOIN + TOK_TABREF + TOK_TABNAME + src + s1 + TOK_TABREF + TOK_TABNAME + src + s2 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_WHERE + and + false + = + . + TOK_TABLE_OR_COL + s1 + value + . + TOK_TABLE_OR_COL + s2 + value + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: false (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + tag: 1 + value expressions: key (type: string) + auto parallelism: false + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: false (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + tag: 0 + value expressions: key (type: string) + auto parallelism: false + Path -> Alias: + -mr-10003default.src{} [s2, s1] + Path -> Partition: + -mr-10003default.src{} + Partition + base file name: src + input format: org.apache.hadoop.hive.ql.io.OneNullRowInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.NullStructSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.NullStructSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + -mr-10003default.src{} [s2, s1] + Needs Tagging: true + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {VALUE._col0} {KEY.reducesinkkey0} + 1 {VALUE._col0} {KEY.reducesinkkey0} + outputColumnNames: _col0, _col1, _col5, _col6 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types string:string:string:string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * from src s1, src s2 where false and s1.value = s2.value +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select * from src s1, src s2 where false and s1.value = s2.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/orc_predicate_pushdown.q.out b/ql/src/test/results/clientpositive/orc_predicate_pushdown.q.out index bc8d242..359d4ac 100644 --- a/ql/src/test/results/clientpositive/orc_predicate_pushdown.q.out +++ b/ql/src/test/results/clientpositive/orc_predicate_pushdown.q.out @@ -321,12 +321,12 @@ STAGE PLANS: alias: orc_pred Statistics: Num rows: 1049 Data size: 311170 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((t is not null and (t < 0)) and (t > -2)) (type: boolean) - Statistics: Num rows: 58 Data size: 17204 Basic stats: COMPLETE Column stats: NONE + predicate: ((t < 0) and (t > -2)) (type: boolean) + Statistics: Num rows: 116 Data size: 34409 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: t (type: tinyint) outputColumnNames: t - Statistics: Num rows: 58 Data size: 17204 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 116 Data size: 34409 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(t)) mode: hash @@ -380,15 +380,15 @@ STAGE PLANS: Map Operator Tree: TableScan alias: orc_pred - filterExpr: ((t is not null and (t < 0)) and (t > -2)) (type: boolean) + filterExpr: ((t < 0) and (t > -2)) (type: boolean) Statistics: Num rows: 1049 Data size: 311170 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((t is not null and (t < 0)) and (t > -2)) (type: boolean) - Statistics: Num rows: 58 Data size: 17204 Basic stats: COMPLETE Column stats: NONE + predicate: ((t < 0) and (t > -2)) (type: boolean) + Statistics: Num rows: 116 Data size: 34409 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: t (type: tinyint) outputColumnNames: t - Statistics: Num rows: 58 Data size: 17204 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 116 Data size: 34409 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(t)) mode: hash diff --git a/ql/src/test/results/clientpositive/parallel.q.out b/ql/src/test/results/clientpositive/parallel.q.out index f6c0f65..5255ccd 100644 --- a/ql/src/test/results/clientpositive/parallel.q.out +++ b/ql/src/test/results/clientpositive/parallel.q.out @@ -161,7 +161,6 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@src_a POSTHOOK: Output: default@src_b -POSTHOOK: Lineage: src_a.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: src_a.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: src_b.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: src_b.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] diff --git a/ql/src/test/results/clientpositive/partition_boolexpr.q.out b/ql/src/test/results/clientpositive/partition_boolexpr.q.out new file mode 100644 index 0000000..7d414ff --- /dev/null +++ b/ql/src/test/results/clientpositive/partition_boolexpr.q.out @@ -0,0 +1,299 @@ +PREHOOK: query: -- create testing table. +create table part_boolexpr(key int, value string) partitioned by (dt int, ts string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@part_boolexpr +POSTHOOK: query: -- create testing table. +create table part_boolexpr(key int, value string) partitioned by (dt int, ts string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@part_boolexpr +PREHOOK: query: -- both the below queries should return 0 rows +select count(*) from part_boolexpr where key = 'abc' +PREHOOK: type: QUERY +PREHOOK: Input: default@part_boolexpr +#### A masked pattern was here #### +POSTHOOK: query: -- both the below queries should return 0 rows +select count(*) from part_boolexpr where key = 'abc' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_boolexpr +#### A masked pattern was here #### +0 +PREHOOK: query: select * from part_boolexpr where dt = 'abc' +PREHOOK: type: QUERY +PREHOOK: Input: default@part_boolexpr +#### A masked pattern was here #### +POSTHOOK: query: select * from part_boolexpr where dt = 'abc' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_boolexpr +#### A masked pattern was here #### +PREHOOK: query: explain select count(1) from srcpart where true +PREHOOK: type: QUERY +POSTHOOK: query: explain select count(1) from srcpart where true +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select count(1) from srcpart where false +PREHOOK: type: QUERY +POSTHOOK: query: explain select count(1) from srcpart where false +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select count(1) from srcpart where true and hr='11' +PREHOOK: type: QUERY +POSTHOOK: query: explain select count(1) from srcpart where true and hr='11' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select count(1) from srcpart where true or hr='11' +PREHOOK: type: QUERY +POSTHOOK: query: explain select count(1) from srcpart where true or hr='11' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select count(1) from srcpart where false or hr='11' +PREHOOK: type: QUERY +POSTHOOK: query: explain select count(1) from srcpart where false or hr='11' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select count(1) from srcpart where false and hr='11' +PREHOOK: type: QUERY +POSTHOOK: query: explain select count(1) from srcpart where false and hr='11' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + diff --git a/ql/src/test/results/clientpositive/pcr.q.out b/ql/src/test/results/clientpositive/pcr.q.out index 7fe77e0..568417a 100644 --- a/ql/src/test/results/clientpositive/pcr.q.out +++ b/ql/src/test/results/clientpositive/pcr.q.out @@ -2708,7 +2708,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: NONE tag: 1 - value expressions: value (type: string), ds (type: string) + value expressions: value (type: string) auto parallelism: false TableScan alias: t1 @@ -2724,7 +2724,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: NONE tag: 0 - value expressions: value (type: string), ds (type: string) + value expressions: value (type: string) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -2782,13 +2782,13 @@ STAGE PLANS: condition map: Inner Join 0 to 1 condition expressions: - 0 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} - 1 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} - outputColumnNames: _col0, _col1, _col2, _col6, _col7, _col8 + 0 {KEY.reducesinkkey0} {VALUE._col0} + 1 {KEY.reducesinkkey0} {VALUE._col0} + outputColumnNames: _col0, _col1, _col6, _col7 Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col6 (type: int), _col7 (type: string), _col8 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + expressions: _col0 (type: int), _col1 (type: string), _col6 (type: int), _col7 (type: string) + outputColumnNames: _col0, _col1, _col3, _col4 Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -2799,8 +2799,8 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: - columns _col0,_col1,_col2,_col3,_col4,_col5 - columns.types int,string,string,int,string,string + columns _col0,_col1,_col3,_col4 + columns.types int,string,int,string escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -2818,7 +2818,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: int), _col4 (type: string), _col5 (type: string) + value expressions: _col1 (type: string), _col3 (type: int), _col4 (type: string) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -2829,8 +2829,8 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: - columns _col0,_col1,_col2,_col3,_col4,_col5 - columns.types int,string,string,int,string,string + columns _col0,_col1,_col3,_col4 + columns.types int,string,int,string escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -2838,8 +2838,8 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: - columns _col0,_col1,_col2,_col3,_col4,_col5 - columns.types int,string,string,int,string,string + columns _col0,_col1,_col3,_col4 + columns.types int,string,int,string escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -2848,7 +2848,7 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: string), VALUE._col4 (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), '2000-04-08' (type: string), VALUE._col2 (type: int), VALUE._col3 (type: string), '2000-04-08' (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -3005,7 +3005,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: NONE tag: 1 - value expressions: value (type: string), ds (type: string) + value expressions: value (type: string) auto parallelism: false TableScan alias: t1 @@ -3021,7 +3021,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: NONE tag: 0 - value expressions: value (type: string), ds (type: string) + value expressions: value (type: string) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -3125,13 +3125,13 @@ STAGE PLANS: condition map: Inner Join 0 to 1 condition expressions: - 0 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} - 1 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} - outputColumnNames: _col0, _col1, _col2, _col6, _col7, _col8 + 0 {KEY.reducesinkkey0} {VALUE._col0} + 1 {KEY.reducesinkkey0} {VALUE._col0} + outputColumnNames: _col0, _col1, _col6, _col7 Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col6 (type: int), _col7 (type: string), _col8 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + expressions: _col0 (type: int), _col1 (type: string), _col6 (type: int), _col7 (type: string) + outputColumnNames: _col0, _col1, _col3, _col4 Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -3142,8 +3142,8 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: - columns _col0,_col1,_col2,_col3,_col4,_col5 - columns.types int,string,string,int,string,string + columns _col0,_col1,_col3,_col4 + columns.types int,string,int,string escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -3161,7 +3161,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: int), _col4 (type: string), _col5 (type: string) + value expressions: _col1 (type: string), _col3 (type: int), _col4 (type: string) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -3172,8 +3172,8 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: - columns _col0,_col1,_col2,_col3,_col4,_col5 - columns.types int,string,string,int,string,string + columns _col0,_col1,_col3,_col4 + columns.types int,string,int,string escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -3181,8 +3181,8 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: - columns _col0,_col1,_col2,_col3,_col4,_col5 - columns.types int,string,string,int,string,string + columns _col0,_col1,_col3,_col4 + columns.types int,string,int,string escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -3191,7 +3191,7 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: string), VALUE._col4 (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), '2000-04-08' (type: string), VALUE._col2 (type: int), VALUE._col3 (type: string), '2000-04-09' (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -5448,11 +5448,11 @@ STAGE PLANS: predicate: (key = 11) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: value (type: string), ds (type: string), hr (type: string) - outputColumnNames: _col1, _col2, _col3 + expressions: value (type: string), hr (type: string) + outputColumnNames: _col1, _col3 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: '11' (type: string), _col2 (type: string), _col3 (type: string) + key expressions: '11' (type: string), '2008-04-08' (type: string), _col3 (type: string) sort order: +++ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE tag: -1 @@ -5559,7 +5559,7 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: '11' (type: string), VALUE._col0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string) + expressions: '11' (type: string), VALUE._col0 (type: string), '2008-04-08' (type: string), KEY.reducesinkkey2 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -5671,11 +5671,11 @@ STAGE PLANS: predicate: (key = 11) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: value (type: string), ds (type: string), hr (type: string) - outputColumnNames: _col1, _col2, _col3 + expressions: value (type: string), ds (type: string) + outputColumnNames: _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: '11' (type: string), _col2 (type: string), _col3 (type: string) + key expressions: '11' (type: string), _col2 (type: string), '11' (type: string) sort order: +++ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE tag: -1 @@ -5782,7 +5782,7 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: '11' (type: string), VALUE._col0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string) + expressions: '11' (type: string), VALUE._col0 (type: string), KEY.reducesinkkey1 (type: string), '11' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -6067,3 +6067,66 @@ POSTHOOK: query: drop table foo_field POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@foo_field POSTHOOK: Output: default@foo_field +PREHOOK: query: explain select key,value from srcpart where cast(hr as double) = cast(11 as double) +PREHOOK: type: QUERY +POSTHOOK: query: explain select key,value from srcpart where cast(hr as double) = cast(11 as double) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: srcpart + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + ListSink + +PREHOOK: query: explain select key,value from srcpart where hr = cast(11 as double) +PREHOOK: type: QUERY +POSTHOOK: query: explain select key,value from srcpart where hr = cast(11 as double) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: srcpart + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + ListSink + +PREHOOK: query: explain select key,value from srcpart where cast(hr as double) = 11 +PREHOOK: type: QUERY +POSTHOOK: query: explain select key,value from srcpart where cast(hr as double) = 11 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: srcpart + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + ListSink + diff --git a/ql/src/test/results/clientpositive/ppd2.q.out b/ql/src/test/results/clientpositive/ppd2.q.out index 1efce1b..618a545 100644 --- a/ql/src/test/results/clientpositive/ppd2.q.out +++ b/ql/src/test/results/clientpositive/ppd2.q.out @@ -350,22 +350,22 @@ STAGE PLANS: alias: y Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key = 20)) (type: boolean) - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 20) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: '20' (type: string) sort order: + - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE TableScan alias: x Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key = 20)) (type: boolean) - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 20) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: '20' (type: string) sort order: + - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Reduce Operator Tree: Join Operator @@ -375,11 +375,11 @@ STAGE PLANS: 0 {VALUE._col0} 1 outputColumnNames: _col1 - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -395,15 +395,15 @@ STAGE PLANS: key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator expressions: '20' (type: string), KEY.reducesinkkey0 (type: string), '20' (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -455,25 +455,22 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((ds = '2008-04-08') and (hr = '11')) (type: boolean) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value + Group By Operator + aggregations: count(value) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) diff --git a/ql/src/test/results/clientpositive/ppd_clusterby.q.out b/ql/src/test/results/clientpositive/ppd_clusterby.q.out index 7b1abdd..e3643f1 100644 --- a/ql/src/test/results/clientpositive/ppd_clusterby.q.out +++ b/ql/src/test/results/clientpositive/ppd_clusterby.q.out @@ -74,22 +74,22 @@ STAGE PLANS: alias: y Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key = 20)) (type: boolean) - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 20) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: '20' (type: string) sort order: + - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE TableScan alias: x Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key = 20)) (type: boolean) - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 20) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: '20' (type: string) sort order: + - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Reduce Operator Tree: Join Operator @@ -99,11 +99,11 @@ STAGE PLANS: 0 {VALUE._col0} 1 outputColumnNames: _col1 - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -119,15 +119,15 @@ STAGE PLANS: key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator expressions: '20' (type: string), KEY.reducesinkkey0 (type: string), '20' (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -224,22 +224,22 @@ STAGE PLANS: alias: y Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key = 20)) (type: boolean) - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 20) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: '20' (type: string) sort order: + - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE TableScan alias: x Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key = 20)) (type: boolean) - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 20) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: '20' (type: string) sort order: + - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Reduce Operator Tree: Join Operator @@ -249,11 +249,11 @@ STAGE PLANS: 0 {VALUE._col0} 1 outputColumnNames: _col1 - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -269,15 +269,15 @@ STAGE PLANS: key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator expressions: '20' (type: string), KEY.reducesinkkey0 (type: string), '20' (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/ppd_gby_join.q.out b/ql/src/test/results/clientpositive/ppd_gby_join.q.out index a9d8491..a572726 100644 --- a/ql/src/test/results/clientpositive/ppd_gby_join.q.out +++ b/ql/src/test/results/clientpositive/ppd_gby_join.q.out @@ -31,38 +31,38 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((key > '2') and (key < '400')) and key is not null) and (key <> '4')) and (key > '20')) (type: boolean) - Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE + predicate: ((((key > '2') and (key < '400')) and (key <> '4')) and (key > '20')) (type: boolean) + Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col0 < '400') and _col0 is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 21 Basic stats: COMPLETE Column stats: NONE + predicate: (_col0 < '400') (type: boolean) + Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((((key > '1') and (key < '400')) and key is not null) and (key > '20')) and ((value < 'val_50') or (key > '2'))) and (key <> '4')) (type: boolean) - Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE + predicate: (((((key > '1') and (key < '400')) and (key > '20')) and ((value < 'val_50') or (key > '2'))) and (key <> '4')) (type: boolean) + Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col0 < '400') and _col0 is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + predicate: (_col0 < '400') (type: boolean) + Statistics: Num rows: 4 Data size: 42 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 42 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator @@ -72,7 +72,7 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 23 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 69 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((((_col0 > '20') and ((_col1 < 'val_50') or (_col0 > '2'))) and ((_col2 > '50') or (_col0 < '50'))) and (_col2 <> '4')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE @@ -161,32 +161,32 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((key > '2') and (key < '400')) and key is not null) and (key <> '4')) and (key > '20')) (type: boolean) - Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE + predicate: ((((key > '2') and (key < '400')) and (key <> '4')) and (key > '20')) (type: boolean) + Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((((key > '1') and (key < '400')) and key is not null) and (key > '20')) and ((value < 'val_50') or (key > '2'))) and (key <> '4')) (type: boolean) - Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE + predicate: (((((key > '1') and (key < '400')) and (key > '20')) and ((value < 'val_50') or (key > '2'))) and (key <> '4')) (type: boolean) + Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator @@ -196,20 +196,20 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 9 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 19 Data size: 210 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((((_col0 > '20') and ((_col1 < 'val_50') or (_col0 > '2'))) and ((_col2 > '50') or (_col0 < '50'))) and (_col2 <> '4')) (type: boolean) - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -225,7 +225,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -233,14 +233,14 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/ppd_join.q.out b/ql/src/test/results/clientpositive/ppd_join.q.out index a689b09..413e899 100644 --- a/ql/src/test/results/clientpositive/ppd_join.q.out +++ b/ql/src/test/results/clientpositive/ppd_join.q.out @@ -28,39 +28,39 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((key > '2') and (key < '400')) and key is not null) and (key <> '4')) and (key > '20')) (type: boolean) - Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE + predicate: ((((key > '2') and (key < '400')) and (key <> '4')) and (key > '20')) (type: boolean) + Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col0 < '400') and _col0 is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 21 Basic stats: COMPLETE Column stats: NONE + predicate: (_col0 < '400') (type: boolean) + Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((((key > '1') and (key < '400')) and key is not null) and (key > '20')) and ((value < 'val_50') or (key > '2'))) and (key <> '4')) (type: boolean) - Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE + predicate: (((((key > '1') and (key < '400')) and (key > '20')) and ((value < 'val_50') or (key > '2'))) and (key <> '4')) (type: boolean) + Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col0 < '400') and _col0 is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + predicate: (_col0 < '400') (type: boolean) + Statistics: Num rows: 4 Data size: 42 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 42 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator @@ -70,7 +70,7 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2 Data size: 23 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 69 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((((_col0 > '20') and ((_col1 < 'val_50') or (_col0 > '2'))) and ((_col2 > '50') or (_col0 < '50'))) and (_col2 <> '4')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE @@ -560,33 +560,33 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((key > '2') and (key < '400')) and key is not null) and (key <> '4')) and (key > '20')) (type: boolean) - Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE + predicate: ((((key > '2') and (key < '400')) and (key <> '4')) and (key > '20')) (type: boolean) + Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((((key > '1') and (key < '400')) and key is not null) and (key > '20')) and ((value < 'val_50') or (key > '2'))) and (key <> '4')) (type: boolean) - Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE + predicate: (((((key > '1') and (key < '400')) and (key > '20')) and ((value < 'val_50') or (key > '2'))) and (key <> '4')) (type: boolean) + Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator @@ -596,17 +596,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 9 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 19 Data size: 210 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((((_col0 > '20') and ((_col1 < 'val_50') or (_col0 > '2'))) and ((_col2 > '50') or (_col0 < '50'))) and (_col2 <> '4')) (type: boolean) - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/ppd_join2.q.out b/ql/src/test/results/clientpositive/ppd_join2.q.out index 39c48be..e5c1ee5 100644 --- a/ql/src/test/results/clientpositive/ppd_join2.q.out +++ b/ql/src/test/results/clientpositive/ppd_join2.q.out @@ -35,39 +35,39 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((key <> '305') and (key < '400')) and key is not null) and (key <> '14')) and (key <> '311')) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: ((((key <> '305') and (key < '400')) and (key <> '14')) and (key <> '311')) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col0 < '400') and _col0 is not null) (type: boolean) - Statistics: Num rows: 14 Data size: 148 Basic stats: COMPLETE Column stats: NONE + predicate: (_col0 < '400') (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 14 Data size: 148 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((((key <> '302') and (key < '400')) and key is not null) and value is not null) and (key <> '311')) and ((value <> 'val_50') or (key > '1'))) and (key <> '14')) (type: boolean) - Statistics: Num rows: 56 Data size: 594 Basic stats: COMPLETE Column stats: NONE + predicate: ((((((key <> '302') and (key < '400')) and value is not null) and (key <> '311')) and ((value <> 'val_50') or (key > '1'))) and (key <> '14')) (type: boolean) + Statistics: Num rows: 110 Data size: 1168 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 56 Data size: 594 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 110 Data size: 1168 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col0 < '400') and _col0 is not null) (type: boolean) - Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE + predicate: (_col0 < '400') (type: boolean) + Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator @@ -77,10 +77,10 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 15 Data size: 162 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 60 Data size: 642 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col1 is not null (type: boolean) - Statistics: Num rows: 8 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 30 Data size: 321 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -96,7 +96,7 @@ STAGE PLANS: key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 8 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 30 Data size: 321 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col2 (type: string), _col3 (type: string) TableScan alias: src @@ -1728,33 +1728,33 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((key <> '305') and (key < '400')) and key is not null) and (key <> '14')) and (key <> '311')) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: ((((key <> '305') and (key < '400')) and (key <> '14')) and (key <> '311')) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((((key <> '302') and (key < '400')) and key is not null) and value is not null) and (key <> '311')) and ((value <> 'val_50') or (key > '1'))) and (key <> '14')) (type: boolean) - Statistics: Num rows: 56 Data size: 594 Basic stats: COMPLETE Column stats: NONE + predicate: ((((((key <> '302') and (key < '400')) and value is not null) and (key <> '311')) and ((value <> 'val_50') or (key > '1'))) and (key <> '14')) (type: boolean) + Statistics: Num rows: 110 Data size: 1168 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 56 Data size: 594 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 110 Data size: 1168 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 56 Data size: 594 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 110 Data size: 1168 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator @@ -1764,7 +1764,7 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -1780,7 +1780,7 @@ STAGE PLANS: key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col2 (type: string), _col3 (type: string) TableScan alias: src diff --git a/ql/src/test/results/clientpositive/ppd_join3.q.out b/ql/src/test/results/clientpositive/ppd_join3.q.out index fb4b306..62b21c8 100644 --- a/ql/src/test/results/clientpositive/ppd_join3.q.out +++ b/ql/src/test/results/clientpositive/ppd_join3.q.out @@ -34,58 +34,58 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((((key <> '12') and (key < '400')) and key is not null) and (key <> '4')) and (key > '0')) and (key <> '1')) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: (((((key <> '12') and (key < '400')) and (key <> '4')) and (key > '0')) and (key <> '1')) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col0 < '400') and _col0 is not null) (type: boolean) - Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + predicate: (_col0 < '400') (type: boolean) + Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((((key <> '11') and (key < '400')) and key is not null) and (key > '0')) and ((value <> 'val_500') or (key > '1'))) and (key <> '4')) and (key <> '1')) (type: boolean) - Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE + predicate: ((((((key <> '11') and (key < '400')) and (key > '0')) and ((value <> 'val_500') or (key > '1'))) and (key <> '4')) and (key <> '1')) (type: boolean) + Statistics: Num rows: 73 Data size: 775 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 73 Data size: 775 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col0 < '400') and _col0 is not null) (type: boolean) - Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE + predicate: (_col0 < '400') (type: boolean) + Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((((key <> '13') and (key < '400')) and key is not null) and (key <> '1')) and (key > '0')) and (key <> '4')) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: (((((key <> '13') and (key < '400')) and (key <> '1')) and (key > '0')) and (key <> '4')) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col0 < '400') and _col0 is not null) (type: boolean) - Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + predicate: (_col0 < '400') (type: boolean) + Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -96,17 +96,17 @@ STAGE PLANS: 1 {KEY.reducesinkkey0} {VALUE._col0} 2 {KEY.reducesinkkey0} outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 52 Data size: 558 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((((_col0 > '0') and ((_col1 <> 'val_500') or (_col0 > '1'))) and ((_col2 > '10') or (_col0 <> '10'))) and (_col2 <> '4')) and (_col4 <> '1')) (type: boolean) - Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 311 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 311 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 311 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1755,49 +1755,49 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((((key <> '12') and (key < '400')) and key is not null) and (key <> '4')) and (key > '0')) and (key <> '1')) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: (((((key <> '12') and (key < '400')) and (key <> '4')) and (key > '0')) and (key <> '1')) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((((key <> '11') and (key < '400')) and key is not null) and (key > '0')) and ((value <> 'val_500') or (key > '1'))) and (key <> '4')) and (key <> '1')) (type: boolean) - Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE + predicate: ((((((key <> '11') and (key < '400')) and (key > '0')) and ((value <> 'val_500') or (key > '1'))) and (key <> '4')) and (key <> '1')) (type: boolean) + Statistics: Num rows: 73 Data size: 775 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 73 Data size: 775 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 73 Data size: 775 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((((key <> '13') and (key < '400')) and key is not null) and (key <> '1')) and (key > '0')) and (key <> '4')) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: (((((key <> '13') and (key < '400')) and (key <> '1')) and (key > '0')) and (key <> '4')) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -1808,17 +1808,17 @@ STAGE PLANS: 1 {KEY.reducesinkkey0} {VALUE._col0} 2 {KEY.reducesinkkey0} outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 79 Data size: 840 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 160 Data size: 1705 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((((_col0 > '0') and ((_col1 <> 'val_500') or (_col0 > '1'))) and ((_col2 > '10') or (_col0 <> '10'))) and (_col2 <> '4')) and (_col4 <> '1')) (type: boolean) - Statistics: Num rows: 45 Data size: 478 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 93 Data size: 991 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 45 Data size: 478 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 93 Data size: 991 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 45 Data size: 478 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 93 Data size: 991 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/ppd_join4.q.out b/ql/src/test/results/clientpositive/ppd_join4.q.out index c995f60..de7923a 100644 --- a/ql/src/test/results/clientpositive/ppd_join4.q.out +++ b/ql/src/test/results/clientpositive/ppd_join4.q.out @@ -82,7 +82,7 @@ STAGE PLANS: alias: t3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Filter Operator - predicate: (id is not null and (id = 'a')) (type: boolean) + predicate: (id = 'a') (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: 'a' (type: string) diff --git a/ql/src/test/results/clientpositive/ppd_random.q.out b/ql/src/test/results/clientpositive/ppd_random.q.out index 9463fd8..f21054a 100644 --- a/ql/src/test/results/clientpositive/ppd_random.q.out +++ b/ql/src/test/results/clientpositive/ppd_random.q.out @@ -28,20 +28,20 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > '2') and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key > '2') (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 42 Data size: 445 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 42 Data size: 445 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan alias: src @@ -121,17 +121,17 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > '2') and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key > '2') (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan alias: src diff --git a/ql/src/test/results/clientpositive/ppd_udf_case.q.out b/ql/src/test/results/clientpositive/ppd_udf_case.q.out index e24be0a..1826993 100644 --- a/ql/src/test/results/clientpositive/ppd_udf_case.q.out +++ b/ql/src/test/results/clientpositive/ppd_udf_case.q.out @@ -44,7 +44,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string), ds (type: string), hr (type: string) + value expressions: value (type: string), hr (type: string) TableScan alias: a Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE @@ -56,23 +56,23 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string), ds (type: string), hr (type: string) + value expressions: value (type: string), hr (type: string) Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 condition expressions: - 0 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} {VALUE._col2} - 1 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} {VALUE._col2} - outputColumnNames: _col0, _col1, _col2, _col3, _col7, _col8, _col9, _col10 + 0 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col2} + 1 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col2} + outputColumnNames: _col0, _col1, _col3, _col7, _col8, _col10 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((_col2 = '2008-04-08') and (_col9 = '2008-04-08')) and CASE (_col0) WHEN ('27') THEN (true) WHEN ('38') THEN (false) ELSE (null) END) (type: boolean) - Statistics: Num rows: 34 Data size: 361 Basic stats: COMPLETE Column stats: NONE + predicate: CASE (_col0) WHEN ('27') THEN (true) WHEN ('38') THEN (false) ELSE (null) END (type: boolean) + Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string), _col7 (type: string), _col8 (type: string), _col10 (type: string) outputColumnNames: _col0, _col1, _col3, _col4, _col5, _col7 - Statistics: Num rows: 34 Data size: 361 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -87,15 +87,15 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), '2008-04-08' (type: string), _col7 (type: string) sort order: ++++++++ - Statistics: Num rows: 34 Data size: 361 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), '2008-04-08' (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string), '2008-04-08' (type: string), KEY.reducesinkkey7 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 34 Data size: 361 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 34 Data size: 361 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -189,7 +189,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string), ds (type: string), hr (type: string) + value expressions: value (type: string), hr (type: string) TableScan alias: a Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE @@ -201,19 +201,19 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string), ds (type: string), hr (type: string) + value expressions: value (type: string), hr (type: string) Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 condition expressions: - 0 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} {VALUE._col2} - 1 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} {VALUE._col2} - outputColumnNames: _col0, _col1, _col2, _col3, _col7, _col8, _col9, _col10 + 0 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col2} + 1 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col2} + outputColumnNames: _col0, _col1, _col3, _col7, _col8, _col10 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: string), _col10 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string), _col7 (type: string), _col8 (type: string), _col10 (type: string) + outputColumnNames: _col0, _col1, _col3, _col4, _col5, _col7 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -227,12 +227,12 @@ STAGE PLANS: Map Operator Tree: TableScan Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string) + key expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), '2008-04-08' (type: string), _col7 (type: string) sort order: ++++++++ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string), KEY.reducesinkkey6 (type: string), KEY.reducesinkkey7 (type: string) + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), '2008-04-08' (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string), '2008-04-08' (type: string), KEY.reducesinkkey7 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/ppd_udf_col.q.out b/ql/src/test/results/clientpositive/ppd_udf_col.q.out index 5c38154..64ebd5b 100644 --- a/ql/src/test/results/clientpositive/ppd_udf_col.q.out +++ b/ql/src/test/results/clientpositive/ppd_udf_col.q.out @@ -130,24 +130,24 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (key = 100) (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + predicate: false (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE Select Operator expressions: rand() (type: double), '4' (type: string) outputColumnNames: _col2, _col3 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE Filter Operator predicate: (_col3 <= 3) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE Select Operator expressions: '100' (type: string), _col2 (type: double), _col3 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -333,28 +333,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (key = 100) (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + predicate: false (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE Select Operator - expressions: rand() (type: double), '4' (type: string) - outputColumnNames: _col2, _col3 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (_col3 <= 3) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: '100' (type: string), _col2 (type: double), _col3 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + expressions: '100' (type: string), rand() (type: double), '4' (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/ppd_union_view.q.out b/ql/src/test/results/clientpositive/ppd_union_view.q.out index e41959e..d5863ef 100644 --- a/ql/src/test/results/clientpositive/ppd_union_view.q.out +++ b/ql/src/test/results/clientpositive/ppd_union_view.q.out @@ -177,9 +177,9 @@ STAGE PLANS: predicate: keymap is not null (type: boolean) Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: keymap (type: string), ds (type: string) + key expressions: keymap (type: string), '2011-10-13' (type: string) sort order: ++ - Map-reduce partition columns: keymap (type: string), ds (type: string) + Map-reduce partition columns: keymap (type: string) Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: key (type: string) @@ -193,9 +193,9 @@ STAGE PLANS: predicate: keymap is not null (type: boolean) Statistics: Num rows: 1 Data size: 14 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: keymap (type: string), ds (type: string) + key expressions: keymap (type: string), '2011-10-13' (type: string) sort order: ++ - Map-reduce partition columns: keymap (type: string), ds (type: string) + Map-reduce partition columns: keymap (type: string) Statistics: Num rows: 1 Data size: 14 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: string) @@ -302,13 +302,13 @@ STAGE PLANS: condition map: Inner Join 0 to 1 condition expressions: - 0 {VALUE._col0} {KEY.reducesinkkey1} + 0 {VALUE._col0} 1 {VALUE._col0} - outputColumnNames: _col1, _col2, _col6 + outputColumnNames: _col1, _col6 Statistics: Num rows: 1 Data size: 15 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col6 (type: string), _col1 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1, _col2 + expressions: _col6 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 15 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -319,8 +319,8 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: - columns _col0,_col1,_col2 - columns.types string,string,string + columns _col0,_col1 + columns.types string,string escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -336,7 +336,7 @@ STAGE PLANS: Union Statistics: Num rows: 1 Data size: 15 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + expressions: _col0 (type: string), _col1 (type: string), '2011-10-13' (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 15 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -369,8 +369,8 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: - columns _col0,_col1,_col2 - columns.types string,string,string + columns _col0,_col1 + columns.types string,string escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -378,8 +378,8 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: - columns _col0,_col1,_col2 - columns.types string,string,string + columns _col0,_col1 + columns.types string,string escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -476,8 +476,8 @@ STAGE PLANS: outputColumnNames: _col1, _col6 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: _col6 (type: string), _col1 (type: string), '2011-10-15' (type: string) - outputColumnNames: _col0, _col1, _col2 + expressions: _col6 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false @@ -488,8 +488,8 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: - columns _col0,_col1,_col2 - columns.types string,string,string + columns _col0,_col1 + columns.types string,string escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -505,7 +505,7 @@ STAGE PLANS: Union Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + expressions: _col0 (type: string), _col1 (type: string), '2011-10-15' (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -534,13 +534,13 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE GatherStats: false Select Operator - expressions: key (type: string), value (type: string), ds (type: string) - outputColumnNames: _col0, _col1, _col2 + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE Union Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + expressions: _col0 (type: string), _col1 (type: string), '2011-10-15' (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -573,8 +573,8 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: - columns _col0,_col1,_col2 - columns.types string,string,string + columns _col0,_col1 + columns.types string,string escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -582,8 +582,8 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: - columns _col0,_col1,_col2 - columns.types string,string,string + columns _col0,_col1 + columns.types string,string escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe diff --git a/ql/src/test/results/clientpositive/ptf_matchpath.q.out b/ql/src/test/results/clientpositive/ptf_matchpath.q.out index f04c998..fc0187b 100644 --- a/ql/src/test/results/clientpositive/ptf_matchpath.q.out +++ b/ql/src/test/results/clientpositive/ptf_matchpath.q.out @@ -107,3 +107,27 @@ Baltimore 1142 2010 10 21 5 21 Baltimore 1142 2010 10 22 4 22 Baltimore 1142 2010 10 25 3 25 Baltimore 1142 2010 10 26 2 26 +PREHOOK: query: -- 3. empty partition. +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + (select * from flights_tiny where fl_num = -1142) flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +POSTHOOK: query: -- 3. empty partition. +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + (select * from flights_tiny where fl_num = -1142) flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny +#### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/ql_rewrite_gbtoidx.q.out b/ql/src/test/results/clientpositive/ql_rewrite_gbtoidx.q.out index f7da06c..2c69a86 100644 --- a/ql/src/test/results/clientpositive/ql_rewrite_gbtoidx.q.out +++ b/ql/src/test/results/clientpositive/ql_rewrite_gbtoidx.q.out @@ -257,7 +257,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__lineitem_lineitem_lshipdate_idx__ + alias: default.default__lineitem_lineitem_lshipdate_idx__ Statistics: Num rows: 95 Data size: 8675 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: l_shipdate (type: string), _count_of_l_shipdate (type: bigint) @@ -584,7 +584,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__lineitem_lineitem_lshipdate_idx__ + alias: default.default__lineitem_lineitem_lshipdate_idx__ Statistics: Num rows: 95 Data size: 8675 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: l_shipdate (type: string), _count_of_l_shipdate (type: bigint) @@ -762,10 +762,10 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: lastyear:default__lineitem_lineitem_lshipdate_idx__ + alias: thisyear:default.default__lineitem_lineitem_lshipdate_idx__ Statistics: Num rows: 95 Data size: 8675 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (year(l_shipdate) = 1997) (type: boolean) + predicate: (year(l_shipdate) = 1998) (type: boolean) Statistics: Num rows: 47 Data size: 4291 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: l_shipdate (type: string), _count_of_l_shipdate (type: bigint) @@ -846,10 +846,10 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: thisyear:default__lineitem_lineitem_lshipdate_idx__ + alias: lastyear:default.default__lineitem_lineitem_lshipdate_idx__ Statistics: Num rows: 95 Data size: 8675 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (year(l_shipdate) = 1998) (type: boolean) + predicate: (year(l_shipdate) = 1997) (type: boolean) Statistics: Num rows: 47 Data size: 4291 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: l_shipdate (type: string), _count_of_l_shipdate (type: bigint) @@ -916,7 +916,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: null-subquery1:dummy-subquery1:default__lineitem_lineitem_lshipdate_idx__ + alias: null-subquery1:dummy-subquery1:default.default__lineitem_lineitem_lshipdate_idx__ Statistics: Num rows: 95 Data size: 8675 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: l_shipdate (type: string), _count_of_l_shipdate (type: bigint) @@ -1095,7 +1095,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__tbl_tbl_key_idx__ + alias: default.default__tbl_tbl_key_idx__ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: key (type: int), _count_of_key (type: bigint) @@ -2148,7 +2148,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: int) + expressions: _col0 (type: int), 2 (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator @@ -2364,7 +2364,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__tblpart_tbl_part_index__ + alias: default.default__tblpart_tbl_part_index__ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (key < 10) (type: boolean) @@ -2565,7 +2565,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__tbl_tbl_key_idx__ + alias: default.default__tbl_tbl_key_idx__ Statistics: Num rows: 6 Data size: 430 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), _count_of_key (type: bigint) diff --git a/ql/src/test/results/clientpositive/quotedid_basic.q.out b/ql/src/test/results/clientpositive/quotedid_basic.q.out index 46ec84b..725a0fc 100644 --- a/ql/src/test/results/clientpositive/quotedid_basic.q.out +++ b/ql/src/test/results/clientpositive/quotedid_basic.q.out @@ -121,7 +121,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + expressions: _col0 (type: string), _col1 (type: string), '1' (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator @@ -191,18 +191,17 @@ STAGE PLANS: Map Operator Tree: TableScan Reduce Output Operator - key expressions: _col2 (type: string), _col1 (type: string) + key expressions: '1' (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + value expressions: _col0 (type: string), _col1 (type: string), '1' (type: string) Reduce Operator Tree: Extract Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE PTF Operator Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _wcol0 (type: int) + expressions: _col0 (type: string), _col1 (type: string), '1' (type: string), _wcol0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator @@ -274,18 +273,17 @@ STAGE PLANS: Map Operator Tree: TableScan Reduce Output Operator - key expressions: _col2 (type: string), _col1 (type: string) + key expressions: '1' (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + value expressions: _col0 (type: string), _col1 (type: string), '1' (type: string) Reduce Operator Tree: Extract Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE PTF Operator Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _wcol0 (type: int) + expressions: _col0 (type: string), _col1 (type: string), '1' (type: string), _wcol0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/quotedid_partition.q.out b/ql/src/test/results/clientpositive/quotedid_partition.q.out index 6253b33..03db1b5 100644 --- a/ql/src/test/results/clientpositive/quotedid_partition.q.out +++ b/ql/src/test/results/clientpositive/quotedid_partition.q.out @@ -46,7 +46,7 @@ STAGE PLANS: predicate: (x+1 = '10') (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: '10' (type: string), y&y (type: string), !@#$%^&*()_q (type: string) + expressions: '10' (type: string), y&y (type: string), 'a' (type: string) outputColumnNames: x+1, y&y, !@#$%^&*()_q Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -66,7 +66,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + expressions: _col0 (type: string), _col1 (type: string), 'a' (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/rand_partitionpruner2.q.out b/ql/src/test/results/clientpositive/rand_partitionpruner2.q.out index 44320e8..971307b 100644 --- a/ql/src/test/results/clientpositive/rand_partitionpruner2.q.out +++ b/ql/src/test/results/clientpositive/rand_partitionpruner2.q.out @@ -74,7 +74,7 @@ STAGE PLANS: predicate: (rand(1) < 0.1) (type: boolean) Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) + expressions: key (type: string), value (type: string), '2008-04-08' (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/rand_partitionpruner3.q.out b/ql/src/test/results/clientpositive/rand_partitionpruner3.q.out index ad2c388..8cfa243 100644 --- a/ql/src/test/results/clientpositive/rand_partitionpruner3.q.out +++ b/ql/src/test/results/clientpositive/rand_partitionpruner3.q.out @@ -73,7 +73,7 @@ STAGE PLANS: predicate: ((rand(1) < 0.1) and (not ((key > 50) or (key < 10)))) (type: boolean) Statistics: Num rows: 56 Data size: 594 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) + expressions: key (type: string), value (type: string), '2008-04-08' (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 56 Data size: 594 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -237,7 +237,7 @@ STAGE PLANS: predicate: (not ((key > 50) or (key < 10))) (type: boolean) Statistics: Num rows: 168 Data size: 1784 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) + expressions: key (type: string), value (type: string), '2008-04-08' (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 168 Data size: 1784 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/regex_col.q.out b/ql/src/test/results/clientpositive/regex_col.q.out index c42ba66..abb6aa7 100644 --- a/ql/src/test/results/clientpositive/regex_col.q.out +++ b/ql/src/test/results/clientpositive/regex_col.q.out @@ -157,24 +157,24 @@ STAGE PLANS: alias: b Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key = 103) and key is not null) (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 103) (type: boolean) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: '103' (type: string), hr (type: string), ds (type: string) sort order: +++ Map-reduce partition columns: hr (type: string), ds (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key = 103) and key is not null) (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 103) (type: boolean) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: '103' (type: string), hr (type: string), ds (type: string) sort order: +++ Map-reduce partition columns: hr (type: string), ds (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -183,11 +183,11 @@ STAGE PLANS: 0 1 {KEY.reducesinkkey2} {KEY.reducesinkkey1} outputColumnNames: _col9, _col10 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col9 (type: string), _col10 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -202,15 +202,15 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/sample1.q.out b/ql/src/test/results/clientpositive/sample1.q.out index 6f8909b..78f2b85 100644 --- a/ql/src/test/results/clientpositive/sample1.q.out +++ b/ql/src/test/results/clientpositive/sample1.q.out @@ -80,7 +80,7 @@ STAGE PLANS: predicate: (((hash(rand()) & 2147483647) % 1) = 0) (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), ds (type: string), hr (type: string) + expressions: UDFToInteger(key) (type: int), value (type: string), '2008-04-08' (type: string), '11' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/sample8.q.out b/ql/src/test/results/clientpositive/sample8.q.out index 69471fe..f42c2d4 100644 --- a/ql/src/test/results/clientpositive/sample8.q.out +++ b/ql/src/test/results/clientpositive/sample8.q.out @@ -98,13 +98,13 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: true - predicate: ((((((hash(key) & 2147483647) % 1) = 0) and key is not null) and value is not null) and (((hash(key) & 2147483647) % 10) = 0)) (type: boolean) - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + predicate: (((((hash(key) & 2147483647) % 1) = 0) and value is not null) and (((hash(key) & 2147483647) % 10) = 0)) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string), value (type: string) sort order: ++ Map-reduce partition columns: key (type: string), value (type: string) - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE tag: 1 auto parallelism: false TableScan @@ -113,15 +113,14 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: true - predicate: ((((((hash(key) & 2147483647) % 10) = 0) and key is not null) and value is not null) and (((hash(key) & 2147483647) % 1) = 0)) (type: boolean) - Statistics: Num rows: 31 Data size: 329 Basic stats: COMPLETE Column stats: NONE + predicate: (((((hash(key) & 2147483647) % 10) = 0) and value is not null) and (((hash(key) & 2147483647) % 1) = 0)) (type: boolean) + Statistics: Num rows: 62 Data size: 658 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string), value (type: string) sort order: ++ Map-reduce partition columns: key (type: string), value (type: string) - Statistics: Num rows: 31 Data size: 329 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 62 Data size: 658 Basic stats: COMPLETE Column stats: NONE tag: 0 - value expressions: ds (type: string), hr (type: string) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -321,24 +320,24 @@ STAGE PLANS: condition map: Inner Join 0 to 1 condition expressions: - 0 {KEY.reducesinkkey0} {KEY.reducesinkkey1} {VALUE._col0} {VALUE._col1} + 0 {KEY.reducesinkkey0} {KEY.reducesinkkey1} 1 {KEY.reducesinkkey0} {KEY.reducesinkkey1} - outputColumnNames: _col0, _col1, _col2, _col3, _col7, _col8 - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col7, _col8 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false - predicate: ((((_col7 = _col0) and (_col8 = _col1)) and (_col2 = '2008-04-08')) and (_col3 = '11')) (type: boolean) - Statistics: Num rows: 8 Data size: 85 Basic stats: COMPLETE Column stats: NONE + predicate: ((_col7 = _col0) and (_col8 = _col1)) (type: boolean) + Statistics: Num rows: 68 Data size: 722 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), '11' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 8 Data size: 85 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 68 Data size: 722 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 8 Data size: 85 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 68 Data size: 722 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git a/ql/src/test/results/clientpositive/select_same_col.q.out b/ql/src/test/results/clientpositive/select_same_col.q.out new file mode 100644 index 0000000..426f716 --- /dev/null +++ b/ql/src/test/results/clientpositive/select_same_col.q.out @@ -0,0 +1,120 @@ +PREHOOK: query: drop table srclimit +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table srclimit +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table srclimit as select * from src limit 10 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@srclimit +POSTHOOK: query: create table srclimit as select * from src limit 10 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@srclimit +PREHOOK: query: select cast(value as binary), value from srclimit +PREHOOK: type: QUERY +PREHOOK: Input: default@srclimit +#### A masked pattern was here #### +POSTHOOK: query: select cast(value as binary), value from srclimit +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srclimit +#### A masked pattern was here #### +val_484 val_484 +val_98 val_98 +val_278 val_278 +val_255 val_255 +val_409 val_409 +val_165 val_165 +val_27 val_27 +val_311 val_311 +val_86 val_86 +val_238 val_238 +PREHOOK: query: select cast(value as binary), value from srclimit order by value +PREHOOK: type: QUERY +PREHOOK: Input: default@srclimit +#### A masked pattern was here #### +POSTHOOK: query: select cast(value as binary), value from srclimit order by value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srclimit +#### A masked pattern was here #### +val_165 val_165 +val_238 val_238 +val_255 val_255 +val_27 val_27 +val_278 val_278 +val_311 val_311 +val_409 val_409 +val_484 val_484 +val_86 val_86 +val_98 val_98 +PREHOOK: query: select cast(value as binary), value from srclimit order by value limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@srclimit +#### A masked pattern was here #### +POSTHOOK: query: select cast(value as binary), value from srclimit order by value limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srclimit +#### A masked pattern was here #### +val_165 val_165 +val_238 val_238 +val_255 val_255 +val_27 val_27 +val_278 val_278 +PREHOOK: query: select cast(value as binary), value, key from srclimit order by value limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@srclimit +#### A masked pattern was here #### +POSTHOOK: query: select cast(value as binary), value, key from srclimit order by value limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srclimit +#### A masked pattern was here #### +val_165 val_165 165 +val_238 val_238 238 +val_255 val_255 255 +val_27 val_27 27 +val_278 val_278 278 +PREHOOK: query: select *, key, value from srclimit +PREHOOK: type: QUERY +PREHOOK: Input: default@srclimit +#### A masked pattern was here #### +POSTHOOK: query: select *, key, value from srclimit +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srclimit +#### A masked pattern was here #### +484 val_484 484 val_484 +98 val_98 98 val_98 +278 val_278 278 val_278 +255 val_255 255 val_255 +409 val_409 409 val_409 +165 val_165 165 val_165 +27 val_27 27 val_27 +311 val_311 311 val_311 +86 val_86 86 val_86 +238 val_238 238 val_238 +PREHOOK: query: select * from (select *, key, value from srclimit) t +PREHOOK: type: QUERY +PREHOOK: Input: default@srclimit +#### A masked pattern was here #### +POSTHOOK: query: select * from (select *, key, value from srclimit) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srclimit +#### A masked pattern was here #### +484 val_484 484 val_484 +98 val_98 98 val_98 +278 val_278 278 val_278 +255 val_255 255 val_255 +409 val_409 409 val_409 +165 val_165 165 val_165 +27 val_27 27 val_27 +311 val_311 311 val_311 +86 val_86 86 val_86 +238 val_238 238 val_238 +PREHOOK: query: drop table srclimit +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@srclimit +PREHOOK: Output: default@srclimit +POSTHOOK: query: drop table srclimit +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@srclimit +POSTHOOK: Output: default@srclimit diff --git a/ql/src/test/results/clientpositive/semijoin.q.out b/ql/src/test/results/clientpositive/semijoin.q.out index a9a9b86..920b2c7 100644 --- a/ql/src/test/results/clientpositive/semijoin.q.out +++ b/ql/src/test/results/clientpositive/semijoin.q.out @@ -451,22 +451,22 @@ STAGE PLANS: alias: b Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 15) and key is not null) (type: boolean) - Statistics: Num rows: 4 Data size: 29 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 15) (type: boolean) + Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: key - Statistics: Num rows: 4 Data size: 29 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: key (type: int), key (type: int) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 29 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: int) sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 29 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE @@ -487,11 +487,11 @@ STAGE PLANS: 0 {VALUE._col0} 1 outputColumnNames: _col1 - Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 56 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 56 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -506,15 +506,15 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: string) sort order: + - Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 56 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 56 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 56 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -681,22 +681,22 @@ STAGE PLANS: alias: t3 Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > 5) and key is not null) (type: boolean) - Statistics: Num rows: 4 Data size: 29 Basic stats: COMPLETE Column stats: NONE + predicate: (key > 5) (type: boolean) + Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 29 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: int) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 29 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 4 Data size: 29 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 51 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -705,11 +705,11 @@ STAGE PLANS: 0 {VALUE._col0} 1 outputColumnNames: _col1 - Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 56 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 56 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -724,15 +724,15 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: string) sort order: + - Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 56 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 56 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 47 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 56 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -876,22 +876,22 @@ STAGE PLANS: alias: t1 Statistics: Num rows: 11 Data size: 79 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > 2) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key > 2) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: int) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 11 Data size: 84 Basic stats: COMPLETE Column stats: NONE diff --git a/ql/src/test/results/clientpositive/show_functions.q.out b/ql/src/test/results/clientpositive/show_functions.q.out index 775cc74..944c567 100644 --- a/ql/src/test/results/clientpositive/show_functions.q.out +++ b/ql/src/test/results/clientpositive/show_functions.q.out @@ -280,3 +280,75 @@ PREHOOK: query: SHOW FUNCTIONS '***' PREHOOK: type: SHOWFUNCTIONS POSTHOOK: query: SHOW FUNCTIONS '***' POSTHOOK: type: SHOWFUNCTIONS +PREHOOK: query: SHOW FUNCTIONS LIKE 'When' +PREHOOK: type: SHOWFUNCTIONS +POSTHOOK: query: SHOW FUNCTIONS LIKE 'When' +POSTHOOK: type: SHOWFUNCTIONS +when +PREHOOK: query: SHOW FUNCTIONS LIKE 'max|min' +PREHOOK: type: SHOWFUNCTIONS +POSTHOOK: query: SHOW FUNCTIONS LIKE 'max|min' +POSTHOOK: type: SHOWFUNCTIONS +max +min +PREHOOK: query: SHOW FUNCTIONS LIKE 'xpath*|m*' +PREHOOK: type: SHOWFUNCTIONS +POSTHOOK: query: SHOW FUNCTIONS LIKE 'xpath*|m*' +POSTHOOK: type: SHOWFUNCTIONS +map +map_keys +map_values +matchpath +max +min +minute +month +xpath +xpath_boolean +xpath_double +xpath_float +xpath_int +xpath_long +xpath_number +xpath_short +xpath_string +PREHOOK: query: SHOW FUNCTIONS LIKE 'nomatch' +PREHOOK: type: SHOWFUNCTIONS +POSTHOOK: query: SHOW FUNCTIONS LIKE 'nomatch' +POSTHOOK: type: SHOWFUNCTIONS +PREHOOK: query: SHOW FUNCTIONS LIKE "log" +PREHOOK: type: SHOWFUNCTIONS +POSTHOOK: query: SHOW FUNCTIONS LIKE "log" +POSTHOOK: type: SHOWFUNCTIONS +log +PREHOOK: query: SHOW FUNCTIONS LIKE 'log' +PREHOOK: type: SHOWFUNCTIONS +POSTHOOK: query: SHOW FUNCTIONS LIKE 'log' +POSTHOOK: type: SHOWFUNCTIONS +log +PREHOOK: query: SHOW FUNCTIONS LIKE `log` +PREHOOK: type: SHOWFUNCTIONS +POSTHOOK: query: SHOW FUNCTIONS LIKE `log` +POSTHOOK: type: SHOWFUNCTIONS +log +PREHOOK: query: SHOW FUNCTIONS LIKE 'log*' +PREHOOK: type: SHOWFUNCTIONS +POSTHOOK: query: SHOW FUNCTIONS LIKE 'log*' +POSTHOOK: type: SHOWFUNCTIONS +log +log10 +log2 +PREHOOK: query: SHOW FUNCTIONS LIKE "log*" +PREHOOK: type: SHOWFUNCTIONS +POSTHOOK: query: SHOW FUNCTIONS LIKE "log*" +POSTHOOK: type: SHOWFUNCTIONS +log +log10 +log2 +PREHOOK: query: SHOW FUNCTIONS LIKE `log*` +PREHOOK: type: SHOWFUNCTIONS +POSTHOOK: query: SHOW FUNCTIONS LIKE `log*` +POSTHOOK: type: SHOWFUNCTIONS +log +log10 +log2 diff --git a/ql/src/test/results/clientpositive/show_indexes_edge_cases.q.out b/ql/src/test/results/clientpositive/show_indexes_edge_cases.q.out index cc6a405..eb2f3bc 100644 --- a/ql/src/test/results/clientpositive/show_indexes_edge_cases.q.out +++ b/ql/src/test/results/clientpositive/show_indexes_edge_cases.q.out @@ -118,10 +118,10 @@ PREHOOK: query: SHOW INDEXES ON show_idx_full PREHOOK: type: SHOWINDEXES POSTHOOK: query: SHOW INDEXES ON show_idx_full POSTHOOK: type: SHOWINDEXES -idx_1 show_idx_full key default__show_idx_full_idx_1__ compact -idx_2 show_idx_full value1 default__show_idx_full_idx_2__ compact -idx_comment show_idx_full value2 default__show_idx_full_idx_comment__ compact index comment -idx_compound show_idx_full key, value1 default__show_idx_full_idx_compound__ compact +idx_1 show_idx_full key default.default__show_idx_full_idx_1__ compact +idx_2 show_idx_full value1 default.default__show_idx_full_idx_2__ compact +idx_comment show_idx_full value2 default.default__show_idx_full_idx_comment__ compact index comment +idx_compound show_idx_full key, value1 default.default__show_idx_full_idx_compound__ compact PREHOOK: query: EXPLAIN SHOW INDEXES ON show_idx_empty PREHOOK: type: SHOWINDEXES POSTHOOK: query: EXPLAIN SHOW INDEXES ON show_idx_empty diff --git a/ql/src/test/results/clientpositive/show_indexes_syntax.q.out b/ql/src/test/results/clientpositive/show_indexes_syntax.q.out index bc96359..e3cdd09 100644 --- a/ql/src/test/results/clientpositive/show_indexes_syntax.q.out +++ b/ql/src/test/results/clientpositive/show_indexes_syntax.q.out @@ -53,7 +53,7 @@ PREHOOK: query: SHOW INDEX ON show_idx_t1 PREHOOK: type: SHOWINDEXES POSTHOOK: query: SHOW INDEX ON show_idx_t1 POSTHOOK: type: SHOWINDEXES -idx_t1 show_idx_t1 key default__show_idx_t1_idx_t1__ compact +idx_t1 show_idx_t1 key default.default__show_idx_t1_idx_t1__ compact PREHOOK: query: EXPLAIN SHOW INDEXES ON show_idx_t1 PREHOOK: type: SHOWINDEXES @@ -79,7 +79,7 @@ PREHOOK: query: SHOW INDEXES ON show_idx_t1 PREHOOK: type: SHOWINDEXES POSTHOOK: query: SHOW INDEXES ON show_idx_t1 POSTHOOK: type: SHOWINDEXES -idx_t1 show_idx_t1 key default__show_idx_t1_idx_t1__ compact +idx_t1 show_idx_t1 key default.default__show_idx_t1_idx_t1__ compact PREHOOK: query: EXPLAIN SHOW FORMATTED INDEXES ON show_idx_t1 PREHOOK: type: SHOWINDEXES @@ -108,7 +108,7 @@ POSTHOOK: type: SHOWINDEXES idx_name tab_name col_names idx_tab_name idx_type comment -idx_t1 show_idx_t1 key default__show_idx_t1_idx_t1__ compact +idx_t1 show_idx_t1 key default.default__show_idx_t1_idx_t1__ compact PREHOOK: query: DROP TABLE show_idx_t1 PREHOOK: type: DROPTABLE PREHOOK: Input: default@show_idx_t1 diff --git a/ql/src/test/results/clientpositive/skewjoin.q.out b/ql/src/test/results/clientpositive/skewjoin.q.out index 4b30708..deb0a8f 100644 --- a/ql/src/test/results/clientpositive/skewjoin.q.out +++ b/ql/src/test/results/clientpositive/skewjoin.q.out @@ -970,48 +970,48 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key < 100) and key is not null) and (key < 80)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: (((key < 100) and (key < 80)) and key is not null) (type: boolean) + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key < 80) and key is not null) and (key < 100)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 80) and (key < 100)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key < 100) and key is not null) and (key < 80)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 100) and (key < 80)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -1023,11 +1023,11 @@ STAGE PLANS: 2 handleSkewJoin: true outputColumnNames: _col0, _col3 - Statistics: Num rows: 59 Data size: 629 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) outputColumnNames: _col0, _col3 - Statistics: Num rows: 59 Data size: 629 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col0)), sum(hash(_col3)) mode: hash diff --git a/ql/src/test/results/clientpositive/skewjoinopt10.q.out b/ql/src/test/results/clientpositive/skewjoinopt10.q.out index 7ff8235..01a2cd2 100644 --- a/ql/src/test/results/clientpositive/skewjoinopt10.q.out +++ b/ql/src/test/results/clientpositive/skewjoinopt10.q.out @@ -114,6 +114,8 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator SELECT * : (no compute) + expressions: _col0 (type: string), _col1 (type: array) + outputColumnNames: org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc, org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Lateral View Join Operator outputColumnNames: _col0, _col1, _col2 @@ -160,6 +162,8 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator SELECT * : (no compute) + expressions: _col0 (type: string), _col1 (type: array) + outputColumnNames: org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc, org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Lateral View Join Operator outputColumnNames: _col0, _col1, _col2 diff --git a/ql/src/test/results/clientpositive/smb_mapjoin9.q.out b/ql/src/test/results/clientpositive/smb_mapjoin9.q.out index fcf0a58..e566384 100644 --- a/ql/src/test/results/clientpositive/smb_mapjoin9.q.out +++ b/ql/src/test/results/clientpositive/smb_mapjoin9.q.out @@ -314,13 +314,13 @@ STAGE PLANS: Inner Join 0 to 1 condition expressions: 0 {key} - 1 {key} {value} {ds} + 1 {key} {value} keys: 0 key (type: int) 1 key (type: int) - outputColumnNames: _col0, _col6, _col7, _col8 + outputColumnNames: _col0, _col6, _col7 Select Operator - expressions: _col6 (type: int), _col7 (type: string), _col8 (type: string), _col0 (type: int) + expressions: _col6 (type: int), _col7 (type: string), '2010-10-15' (type: string), _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 File Output Operator compressed: false diff --git a/ql/src/test/results/clientpositive/smb_mapjoin_10.q.out b/ql/src/test/results/clientpositive/smb_mapjoin_10.q.out index 08c47e4..181d6a9 100644 --- a/ql/src/test/results/clientpositive/smb_mapjoin_10.q.out +++ b/ql/src/test/results/clientpositive/smb_mapjoin_10.q.out @@ -90,14 +90,14 @@ STAGE PLANS: condition map: Inner Join 0 to 1 condition expressions: - 0 {userid} {pageid} {postid} {type} {ds} - 1 {userid} {pageid} {postid} {type} {ds} + 0 {userid} {pageid} {postid} {type} + 1 {userid} {pageid} {postid} {type} keys: 0 userid (type: int), pageid (type: int), postid (type: int), type (type: string) 1 userid (type: int), pageid (type: int), postid (type: int), type (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col8, _col9, _col10, _col11, _col12 + outputColumnNames: _col0, _col1, _col2, _col3, _col8, _col9, _col10, _col11 Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: string), _col4 (type: string), _col8 (type: int), _col9 (type: int), _col10 (type: int), _col11 (type: string), _col12 (type: string) + expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: string), '1' (type: string), _col8 (type: int), _col9 (type: int), _col10 (type: int), _col11 (type: string), '2' (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 File Output Operator compressed: false diff --git a/ql/src/test/results/clientpositive/smb_mapjoin_14.q.out b/ql/src/test/results/clientpositive/smb_mapjoin_14.q.out index 05f5e00..48fd95d 100644 --- a/ql/src/test/results/clientpositive/smb_mapjoin_14.q.out +++ b/ql/src/test/results/clientpositive/smb_mapjoin_14.q.out @@ -403,12 +403,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -746,12 +746,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 8) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 8) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -1045,12 +1045,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -1140,12 +1140,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 diff --git a/ql/src/test/results/clientpositive/smb_mapjoin_25.q.out b/ql/src/test/results/clientpositive/smb_mapjoin_25.q.out index 140c46c..937b284 100644 --- a/ql/src/test/results/clientpositive/smb_mapjoin_25.q.out +++ b/ql/src/test/results/clientpositive/smb_mapjoin_25.q.out @@ -66,22 +66,22 @@ STAGE PLANS: alias: b Statistics: Num rows: 51 Data size: 206 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key = 5)) (type: boolean) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 5) (type: boolean) + Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: 5 (type: int) sort order: + - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 52 Data size: 208 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key = 5)) (type: boolean) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 5) (type: boolean) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: 5 (type: int) sort order: + - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -89,11 +89,11 @@ STAGE PLANS: condition expressions: 0 1 - Statistics: Num rows: 14 Data size: 57 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 114 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 5 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 14 Data size: 57 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 114 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -109,13 +109,13 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 14 Data size: 57 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 114 Basic stats: COMPLETE Column stats: NONE TableScan Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 15 Data size: 61 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 118 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -124,17 +124,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {KEY.reducesinkkey0} outputColumnNames: _col0, _col1 - Statistics: Num rows: 16 Data size: 67 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 129 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_col1 = 5) (type: boolean) - Statistics: Num rows: 8 Data size: 33 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 62 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), 5 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 8 Data size: 33 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 62 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 8 Data size: 33 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 62 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -147,22 +147,22 @@ STAGE PLANS: alias: d Statistics: Num rows: 55 Data size: 222 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key = 5)) (type: boolean) - Statistics: Num rows: 14 Data size: 56 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 5) (type: boolean) + Statistics: Num rows: 27 Data size: 108 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: 5 (type: int) sort order: + - Statistics: Num rows: 14 Data size: 56 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 108 Basic stats: COMPLETE Column stats: NONE TableScan alias: c Statistics: Num rows: 51 Data size: 206 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key = 5)) (type: boolean) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 5) (type: boolean) + Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: 5 (type: int) sort order: + - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -170,11 +170,11 @@ STAGE PLANS: condition expressions: 0 1 - Statistics: Num rows: 15 Data size: 61 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 118 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 5 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 15 Data size: 61 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 118 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -216,8 +216,8 @@ STAGE PLANS: alias: a Statistics: Num rows: 52 Data size: 208 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key = 5)) (type: boolean) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 5) (type: boolean) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -238,8 +238,8 @@ STAGE PLANS: alias: c Statistics: Num rows: 51 Data size: 206 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key = 5)) (type: boolean) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 5) (type: boolean) + Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 diff --git a/ql/src/test/results/clientpositive/smb_mapjoin_6.q.out b/ql/src/test/results/clientpositive/smb_mapjoin_6.q.out index 17e3d87..ba6b107 100644 --- a/ql/src/test/results/clientpositive/smb_mapjoin_6.q.out +++ b/ql/src/test/results/clientpositive/smb_mapjoin_6.q.out @@ -2466,8 +2466,8 @@ STAGE PLANS: alias: b Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key > 1000)) (type: boolean) - Statistics: Num rows: 83 Data size: 798 Basic stats: COMPLETE Column stats: NONE + predicate: (key > 1000) (type: boolean) + Statistics: Num rows: 166 Data size: 1597 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -2583,8 +2583,8 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key > 1000)) (type: boolean) - Statistics: Num rows: 83 Data size: 798 Basic stats: COMPLETE Column stats: NONE + predicate: (key > 1000) (type: boolean) + Statistics: Num rows: 166 Data size: 1597 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -2692,8 +2692,8 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key > 1000)) (type: boolean) - Statistics: Num rows: 83 Data size: 798 Basic stats: COMPLETE Column stats: NONE + predicate: (key > 1000) (type: boolean) + Statistics: Num rows: 166 Data size: 1597 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 diff --git a/ql/src/test/results/clientpositive/sort_merge_join_desc_1.q.out b/ql/src/test/results/clientpositive/sort_merge_join_desc_1.q.out index 539f247..55ee521 100644 --- a/ql/src/test/results/clientpositive/sort_merge_join_desc_1.q.out +++ b/ql/src/test/results/clientpositive/sort_merge_join_desc_1.q.out @@ -66,8 +66,8 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key < 10)) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 diff --git a/ql/src/test/results/clientpositive/sort_merge_join_desc_8.q.out b/ql/src/test/results/clientpositive/sort_merge_join_desc_8.q.out index b594325..3f23f5a 100644 --- a/ql/src/test/results/clientpositive/sort_merge_join_desc_8.q.out +++ b/ql/src/test/results/clientpositive/sort_merge_join_desc_8.q.out @@ -119,8 +119,8 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key < 10)) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 diff --git a/ql/src/test/results/clientpositive/spark/auto_join11.q.out b/ql/src/test/results/clientpositive/spark/auto_join11.q.out index 998c28b..6988a6e 100644 --- a/ql/src/test/results/clientpositive/spark/auto_join11.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_join11.q.out @@ -32,17 +32,17 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 100) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 100) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Map 4 Map Operator Tree: @@ -50,17 +50,17 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 100) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 100) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -70,11 +70,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {VALUE._col0} outputColumnNames: _col0, _col3 - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) outputColumnNames: _col0, _col3 - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col0,_col3)) mode: hash diff --git a/ql/src/test/results/clientpositive/spark/auto_join12.q.out b/ql/src/test/results/clientpositive/spark/auto_join12.q.out index d2b7993..3558b50 100644 --- a/ql/src/test/results/clientpositive/spark/auto_join12.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_join12.q.out @@ -38,17 +38,17 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key < 100) and key is not null) and (key < 80)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: (((key < 100) and (key < 80)) and key is not null) (type: boolean) + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Map 4 Map Operator Tree: @@ -56,34 +56,34 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key < 80) and key is not null) and (key < 100)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 80) and (key < 100)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Map 5 Map Operator Tree: TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key < 100) and key is not null) and (key < 80)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 100) and (key < 80)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -95,11 +95,11 @@ STAGE PLANS: 1 {VALUE._col0} 2 outputColumnNames: _col0, _col3 - Statistics: Num rows: 59 Data size: 629 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) outputColumnNames: _col0, _col3 - Statistics: Num rows: 59 Data size: 629 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col0,_col3)) mode: hash diff --git a/ql/src/test/results/clientpositive/spark/auto_join13.q.out b/ql/src/test/results/clientpositive/spark/auto_join13.q.out index 78aa01e..6195c01 100644 --- a/ql/src/test/results/clientpositive/spark/auto_join13.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_join13.q.out @@ -39,17 +39,17 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 100) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 100) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Map 5 Map Operator Tree: @@ -57,17 +57,17 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 100) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 100) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Map 6 Map Operator Tree: TableScan @@ -94,15 +94,15 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col2, _col3 - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_col0 + _col2) is not null (type: boolean) - Statistics: Num rows: 46 Data size: 489 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: (_col0 + _col2) (type: double) sort order: + Map-reduce partition columns: (_col0 + _col2) (type: double) - Statistics: Num rows: 46 Data size: 489 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col3 (type: string) Reducer 3 Reduce Operator Tree: @@ -113,11 +113,11 @@ STAGE PLANS: 0 {VALUE._col0} {VALUE._col3} 1 outputColumnNames: _col0, _col3 - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 100 Data size: 1065 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) outputColumnNames: _col0, _col3 - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 100 Data size: 1065 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col0,_col3)) mode: hash diff --git a/ql/src/test/results/clientpositive/spark/auto_join14.q.out b/ql/src/test/results/clientpositive/spark/auto_join14.q.out index e1e6a4b..d6643e2 100644 --- a/ql/src/test/results/clientpositive/spark/auto_join14.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_join14.q.out @@ -51,13 +51,13 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > 100) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key > 100) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator diff --git a/ql/src/test/results/clientpositive/spark/auto_join16.q.out b/ql/src/test/results/clientpositive/spark/auto_join16.q.out index 0b6807d..3913832 100644 --- a/ql/src/test/results/clientpositive/spark/auto_join16.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_join16.q.out @@ -32,24 +32,24 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((key > 10) and (key > 20)) and key is not null) and value is not null) and (value < 200)) (type: boolean) - Statistics: Num rows: 4 Data size: 42 Basic stats: COMPLETE Column stats: NONE + predicate: ((((key > 10) and value is not null) and (key > 20)) and (value < 200)) (type: boolean) + Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 4 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan alias: tab Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((key > 20) and key is not null) and value is not null) and (value < 200)) (type: boolean) + predicate: ((((key > 20) and value is not null) and key is not null) and (value < 200)) (type: boolean) Statistics: Num rows: 14 Data size: 148 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string), value (type: string) diff --git a/ql/src/test/results/clientpositive/spark/auto_join27.q.out b/ql/src/test/results/clientpositive/spark/auto_join27.q.out index 67f5739..651f30f 100644 --- a/ql/src/test/results/clientpositive/spark/auto_join27.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_join27.q.out @@ -42,28 +42,28 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 200) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 200) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: key, value - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: key (type: string), value (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Map 6 Map Operator Tree: TableScan alias: src Filter Operator - predicate: ((key < 200) and key is not null) (type: boolean) + predicate: (key < 200) (type: boolean) Select Operator expressions: key (type: string) outputColumnNames: _col0 @@ -77,17 +77,17 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 200) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 200) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Group By Operator @@ -109,9 +109,9 @@ STAGE PLANS: condition expressions: 0 1 - Statistics: Num rows: 136 Data size: 1447 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 273 Data size: 2908 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 136 Data size: 1447 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 273 Data size: 2908 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) mode: hash diff --git a/ql/src/test/results/clientpositive/spark/auto_join29.q.out b/ql/src/test/results/clientpositive/spark/auto_join29.q.out index 0951b8d..42043d3 100644 --- a/ql/src/test/results/clientpositive/spark/auto_join29.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_join29.q.out @@ -3436,13 +3436,13 @@ STAGE PLANS: alias: src2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key < 10) and key is not null) and (key > 10)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 10) and (key > 10)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Map 4 Map Operator Tree: @@ -3450,13 +3450,13 @@ STAGE PLANS: alias: src3 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key > 10) and key is not null) and (key < 10)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > 10) and (key < 10)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Map 5 Map Operator Tree: @@ -3464,13 +3464,13 @@ STAGE PLANS: alias: src1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key > 10) and key is not null) and (key < 10)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > 10) and (key < 10)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Reducer 2 Reduce Operator Tree: @@ -3483,24 +3483,24 @@ STAGE PLANS: 1 {KEY.reducesinkkey0} {VALUE._col0} 2 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 59 Data size: 629 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 59 Data size: 629 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) sort order: ++++++ - Statistics: Num rows: 59 Data size: 629 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE Reducer 3 Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 59 Data size: 629 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 59 Data size: 629 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/spark/auto_join_reordering_values.q.out b/ql/src/test/results/clientpositive/spark/auto_join_reordering_values.q.out index b9f4666..47db320 100644 --- a/ql/src/test/results/clientpositive/spark/auto_join_reordering_values.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_join_reordering_values.q.out @@ -199,7 +199,7 @@ STAGE PLANS: Map-reduce partition columns: date (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -265,7 +265,7 @@ STAGE PLANS: Map-reduce partition columns: dealid (type: int) Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -331,7 +331,7 @@ STAGE PLANS: Map-reduce partition columns: cityid (type: int) Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -398,7 +398,7 @@ STAGE PLANS: Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: 0 value expressions: dealid (type: int), cityid (type: int), userid (type: int) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -464,7 +464,7 @@ STAGE PLANS: Map-reduce partition columns: userid (type: int) Statistics: Num rows: 49 Data size: 196 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -532,7 +532,7 @@ STAGE PLANS: Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: 0 value expressions: _col3 (type: int), _col4 (type: int), _col9 (type: string) - auto parallelism: true + auto parallelism: false Reducer 3 Needs Tagging: true Reduce Operator Tree: @@ -551,7 +551,7 @@ STAGE PLANS: Statistics: Num rows: 5 Data size: 22 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: _col4 (type: int), _col9 (type: string), _col16 (type: int) - auto parallelism: true + auto parallelism: false Reducer 4 Needs Tagging: true Reduce Operator Tree: @@ -570,7 +570,7 @@ STAGE PLANS: Statistics: Num rows: 5 Data size: 24 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: _col9 (type: string), _col16 (type: int) - auto parallelism: true + auto parallelism: false Reducer 5 Needs Tagging: true Reduce Operator Tree: diff --git a/ql/src/test/results/clientpositive/spark/auto_join_without_localtask.q.out b/ql/src/test/results/clientpositive/spark/auto_join_without_localtask.q.out index a78352c..00ab144 100644 --- a/ql/src/test/results/clientpositive/spark/auto_join_without_localtask.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_join_without_localtask.q.out @@ -300,13 +300,13 @@ STAGE PLANS: alias: b Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key > 100)) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key > 100) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan @@ -343,12 +343,12 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) Reducer 3 Reduce Operator Tree: diff --git a/ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out b/ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out index e64d4fb..3b6182a 100644 --- a/ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out @@ -611,34 +611,34 @@ STAGE PLANS: alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -647,9 +647,9 @@ STAGE PLANS: condition expressions: 0 1 - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -1059,34 +1059,34 @@ STAGE PLANS: alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 8) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 8) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 8) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 8) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -1095,9 +1095,9 @@ STAGE PLANS: condition expressions: 0 1 - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -1311,17 +1311,17 @@ STAGE PLANS: alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan @@ -1437,51 +1437,51 @@ STAGE PLANS: alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Map 5 Map Operator Tree: TableScan alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -1492,9 +1492,9 @@ STAGE PLANS: 0 1 2 - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash diff --git a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_1.q.out b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_1.q.out index 9158d65..3b992d5 100644 --- a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_1.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_1.q.out @@ -169,7 +169,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -288,7 +288,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -486,7 +486,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -556,7 +556,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -803,7 +803,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -873,7 +873,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out index f608cc5..31acdf2 100644 --- a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out @@ -84,7 +84,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) + predicate: (key < 6) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -100,7 +100,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) + predicate: (key < 6) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -117,17 +117,17 @@ STAGE PLANS: alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reducer 3 Reduce Operator Tree: Join Operator @@ -136,9 +136,9 @@ STAGE PLANS: condition expressions: 0 1 - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -239,24 +239,24 @@ STAGE PLANS: alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: key - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() bucketGroup: true keys: key (type: int) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Map 5 Map Operator Tree: @@ -264,17 +264,17 @@ STAGE PLANS: alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Group By Operator @@ -282,16 +282,16 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reducer 3 Reduce Operator Tree: Join Operator @@ -300,9 +300,9 @@ STAGE PLANS: condition expressions: 0 1 - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash diff --git a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_11.q.out b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_11.q.out index 3c26363..59f85e1 100644 --- a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_11.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_11.q.out @@ -165,7 +165,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -282,7 +282,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -487,7 +487,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -604,7 +604,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -808,7 +808,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -925,7 +925,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1144,7 +1144,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1261,7 +1261,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE tag: 2 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1378,7 +1378,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_12.q.out b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_12.q.out index 65e496f..4212281 100644 --- a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_12.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_12.q.out @@ -295,7 +295,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 1 Data size: 170 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -365,7 +365,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE tag: 2 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -484,7 +484,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_2.q.out b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_2.q.out index d1bb7a0..451aedb 100644 --- a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_2.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_2.q.out @@ -149,7 +149,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 1 Data size: 113 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -219,7 +219,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -468,7 +468,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 1 Data size: 113 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -538,7 +538,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_3.q.out b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_3.q.out index d57a1d7..9897e11 100644 --- a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_3.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_3.q.out @@ -149,7 +149,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -219,7 +219,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -466,7 +466,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -585,7 +585,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -783,7 +783,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -902,7 +902,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 29 Data size: 2906 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_4.q.out b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_4.q.out index 8244c50..51d68d8 100644 --- a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_4.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_4.q.out @@ -165,7 +165,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 14 Data size: 1425 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -235,7 +235,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 2 Data size: 226 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -482,7 +482,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 2 Data size: 226 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -601,7 +601,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 14 Data size: 1425 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -799,7 +799,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 2 Data size: 226 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -918,7 +918,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 14 Data size: 1425 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_5.q.out b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_5.q.out index 2ab1bca..ff3a23f 100644 --- a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_5.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_5.q.out @@ -130,7 +130,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 14 Data size: 1425 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -196,7 +196,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 1 Data size: 113 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -384,7 +384,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 1 Data size: 113 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -450,7 +450,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 14 Data size: 1425 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -638,7 +638,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 1 Data size: 113 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -704,7 +704,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 14 Data size: 1425 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_7.q.out b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_7.q.out index 16ef3ae..83739a7 100644 --- a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_7.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_7.q.out @@ -182,7 +182,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -301,7 +301,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 2 Data size: 226 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -550,7 +550,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 2 Data size: 226 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -669,7 +669,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -918,7 +918,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 2 Data size: 226 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1037,7 +1037,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_8.q.out b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_8.q.out index 9fd3e5a..123923e 100644 --- a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_8.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_8.q.out @@ -182,7 +182,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -301,7 +301,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -550,7 +550,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -669,7 +669,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -920,7 +920,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1039,7 +1039,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_9.q.out b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_9.q.out index a7f994f..167aee8 100644 --- a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_9.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_9.q.out @@ -741,34 +741,34 @@ STAGE PLANS: alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -777,9 +777,9 @@ STAGE PLANS: condition expressions: 0 1 - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -1189,34 +1189,34 @@ STAGE PLANS: alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 8) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 8) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 8) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 8) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -1225,9 +1225,9 @@ STAGE PLANS: condition expressions: 0 1 - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -1441,17 +1441,17 @@ STAGE PLANS: alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan @@ -1559,17 +1559,17 @@ STAGE PLANS: alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan @@ -1687,51 +1687,51 @@ STAGE PLANS: alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Map 5 Map Operator Tree: TableScan alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -1742,9 +1742,9 @@ STAGE PLANS: 0 1 2 - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -2649,34 +2649,34 @@ STAGE PLANS: alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -2685,9 +2685,9 @@ STAGE PLANS: condition expressions: 0 1 - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -3097,34 +3097,34 @@ STAGE PLANS: alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 8) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 8) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 8) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 8) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -3133,9 +3133,9 @@ STAGE PLANS: condition expressions: 0 1 - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -3221,17 +3221,17 @@ STAGE PLANS: alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan @@ -3339,17 +3339,17 @@ STAGE PLANS: alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan @@ -3467,51 +3467,51 @@ STAGE PLANS: alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Map 5 Map Operator Tree: TableScan alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -3522,9 +3522,9 @@ STAGE PLANS: 0 1 2 - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash diff --git a/ql/src/test/results/clientpositive/spark/bucket3.q.out b/ql/src/test/results/clientpositive/spark/bucket3.q.out index 019c11a..06edc0f 100644 --- a/ql/src/test/results/clientpositive/spark/bucket3.q.out +++ b/ql/src/test/results/clientpositive/spark/bucket3.q.out @@ -239,7 +239,7 @@ STAGE PLANS: predicate: (((hash(key) & 2147483647) % 2) = 0) (type: boolean) Statistics: Num rows: 27 Data size: 2853 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: int), value (type: string), ds (type: string) + expressions: key (type: int), value (type: string), '1' (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 27 Data size: 2853 Basic stats: COMPLETE Column stats: NONE ListSink diff --git a/ql/src/test/results/clientpositive/spark/bucket_map_join_1.q.out b/ql/src/test/results/clientpositive/spark/bucket_map_join_1.q.out index 4ec619e..e14b5b0 100644 --- a/ql/src/test/results/clientpositive/spark/bucket_map_join_1.q.out +++ b/ql/src/test/results/clientpositive/spark/bucket_map_join_1.q.out @@ -131,7 +131,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string), value (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -201,7 +201,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string), value (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/bucket_map_join_2.q.out b/ql/src/test/results/clientpositive/spark/bucket_map_join_2.q.out index 1c288c2..7a0f070 100644 --- a/ql/src/test/results/clientpositive/spark/bucket_map_join_2.q.out +++ b/ql/src/test/results/clientpositive/spark/bucket_map_join_2.q.out @@ -131,7 +131,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string), value (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -201,7 +201,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string), value (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out b/ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out index 60e2a4c..1c14050 100644 --- a/ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out +++ b/ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out @@ -332,34 +332,34 @@ STAGE PLANS: alias: tab_part Statistics: Num rows: 1453 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > 1) and key is not null) (type: boolean) - Statistics: Num rows: 242 Data size: 968 Basic stats: COMPLETE Column stats: NONE + predicate: (key > 1) (type: boolean) + Statistics: Num rows: 484 Data size: 1936 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 242 Data size: 968 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 484 Data size: 1936 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 242 Data size: 968 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 484 Data size: 1936 Basic stats: COMPLETE Column stats: NONE Map 3 Map Operator Tree: TableScan alias: tab_part Statistics: Num rows: 1453 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > 2) and key is not null) (type: boolean) - Statistics: Num rows: 242 Data size: 968 Basic stats: COMPLETE Column stats: NONE + predicate: (key > 2) (type: boolean) + Statistics: Num rows: 484 Data size: 1936 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 242 Data size: 968 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 484 Data size: 1936 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 242 Data size: 968 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 484 Data size: 1936 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -369,14 +369,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {KEY.reducesinkkey0} outputColumnNames: _col0, _col1 - Statistics: Num rows: 266 Data size: 1064 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 532 Data size: 2129 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 266 Data size: 1064 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 532 Data size: 2129 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 266 Data size: 1064 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 532 Data size: 2129 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/spark/bucketmapjoin1.q.out b/ql/src/test/results/clientpositive/spark/bucketmapjoin1.q.out index bea92f9..9da0e8c 100644 --- a/ql/src/test/results/clientpositive/spark/bucketmapjoin1.q.out +++ b/ql/src/test/results/clientpositive/spark/bucketmapjoin1.q.out @@ -485,7 +485,7 @@ STAGE PLANS: Statistics: Num rows: 28 Data size: 2958 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -555,7 +555,7 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 1375 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -878,7 +878,7 @@ STAGE PLANS: Statistics: Num rows: 28 Data size: 2958 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -948,7 +948,7 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 1375 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/bucketmapjoin10.q.out b/ql/src/test/results/clientpositive/spark/bucketmapjoin10.q.out index 8be3edd..0464adb 100644 --- a/ql/src/test/results/clientpositive/spark/bucketmapjoin10.q.out +++ b/ql/src/test/results/clientpositive/spark/bucketmapjoin10.q.out @@ -219,7 +219,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 869 Data size: 3477 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -336,7 +336,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 869 Data size: 3477 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/bucketmapjoin11.q.out b/ql/src/test/results/clientpositive/spark/bucketmapjoin11.q.out index 9e45843..6ed7e8c 100644 --- a/ql/src/test/results/clientpositive/spark/bucketmapjoin11.q.out +++ b/ql/src/test/results/clientpositive/spark/bucketmapjoin11.q.out @@ -229,7 +229,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 1070 Data size: 4281 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -346,7 +346,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 1070 Data size: 4281 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -632,7 +632,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int), part (type: string) Statistics: Num rows: 1070 Data size: 4281 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -749,7 +749,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int), part (type: string) Statistics: Num rows: 1070 Data size: 4281 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/bucketmapjoin12.q.out b/ql/src/test/results/clientpositive/spark/bucketmapjoin12.q.out index 0c1ac4b..8f52751 100644 --- a/ql/src/test/results/clientpositive/spark/bucketmapjoin12.q.out +++ b/ql/src/test/results/clientpositive/spark/bucketmapjoin12.q.out @@ -188,7 +188,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 344 Data size: 1377 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -256,7 +256,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 344 Data size: 1377 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -484,7 +484,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 344 Data size: 1377 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -552,7 +552,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 344 Data size: 1377 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/bucketmapjoin13.q.out b/ql/src/test/results/clientpositive/spark/bucketmapjoin13.q.out index dc1b8cf..7b23cb7 100644 --- a/ql/src/test/results/clientpositive/spark/bucketmapjoin13.q.out +++ b/ql/src/test/results/clientpositive/spark/bucketmapjoin13.q.out @@ -150,7 +150,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 727 Data size: 2908 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -219,7 +219,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 1453 Data size: 5812 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -490,7 +490,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 727 Data size: 2908 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -559,7 +559,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 727 Data size: 2908 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -781,7 +781,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 727 Data size: 2908 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -850,7 +850,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 727 Data size: 2908 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1074,7 +1074,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 727 Data size: 2908 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1143,7 +1143,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 727 Data size: 2908 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/bucketmapjoin2.q.out b/ql/src/test/results/clientpositive/spark/bucketmapjoin2.q.out index 369b7fa..1e6efb6 100644 --- a/ql/src/test/results/clientpositive/spark/bucketmapjoin2.q.out +++ b/ql/src/test/results/clientpositive/spark/bucketmapjoin2.q.out @@ -187,7 +187,7 @@ STAGE PLANS: Statistics: Num rows: 15 Data size: 1583 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -257,7 +257,7 @@ STAGE PLANS: Statistics: Num rows: 28 Data size: 2958 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -589,7 +589,7 @@ STAGE PLANS: Statistics: Num rows: 15 Data size: 1583 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -659,7 +659,7 @@ STAGE PLANS: Statistics: Num rows: 28 Data size: 2958 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1013,7 +1013,7 @@ STAGE PLANS: Statistics: Num rows: 29 Data size: 3062 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1131,7 +1131,7 @@ STAGE PLANS: Statistics: Num rows: 28 Data size: 2958 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/bucketmapjoin3.q.out b/ql/src/test/results/clientpositive/spark/bucketmapjoin3.q.out index 85ffbc9..48fa4a1 100644 --- a/ql/src/test/results/clientpositive/spark/bucketmapjoin3.q.out +++ b/ql/src/test/results/clientpositive/spark/bucketmapjoin3.q.out @@ -218,7 +218,7 @@ STAGE PLANS: Statistics: Num rows: 28 Data size: 2958 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -288,7 +288,7 @@ STAGE PLANS: Statistics: Num rows: 15 Data size: 1583 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -627,7 +627,7 @@ STAGE PLANS: Statistics: Num rows: 28 Data size: 2958 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -697,7 +697,7 @@ STAGE PLANS: Statistics: Num rows: 15 Data size: 1583 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/bucketmapjoin4.q.out b/ql/src/test/results/clientpositive/spark/bucketmapjoin4.q.out index d11b9f4..dff4924 100644 --- a/ql/src/test/results/clientpositive/spark/bucketmapjoin4.q.out +++ b/ql/src/test/results/clientpositive/spark/bucketmapjoin4.q.out @@ -204,7 +204,7 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 1375 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -269,7 +269,7 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 1375 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -577,7 +577,7 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 1375 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -642,7 +642,7 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 1375 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/bucketmapjoin5.q.out b/ql/src/test/results/clientpositive/spark/bucketmapjoin5.q.out index 1bbc30c..35ec7b9 100644 --- a/ql/src/test/results/clientpositive/spark/bucketmapjoin5.q.out +++ b/ql/src/test/results/clientpositive/spark/bucketmapjoin5.q.out @@ -254,7 +254,7 @@ STAGE PLANS: Statistics: Num rows: 55 Data size: 5812 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -372,7 +372,7 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 1375 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -692,7 +692,7 @@ STAGE PLANS: Statistics: Num rows: 29 Data size: 3062 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -810,7 +810,7 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 1375 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/bucketmapjoin7.q.out b/ql/src/test/results/clientpositive/spark/bucketmapjoin7.q.out index 045f615..bd5a64c 100644 --- a/ql/src/test/results/clientpositive/spark/bucketmapjoin7.q.out +++ b/ql/src/test/results/clientpositive/spark/bucketmapjoin7.q.out @@ -154,7 +154,7 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 1375 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -224,7 +224,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 344 Data size: 1377 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/bucketmapjoin8.q.out b/ql/src/test/results/clientpositive/spark/bucketmapjoin8.q.out index 6d72fdf..10d308a 100644 --- a/ql/src/test/results/clientpositive/spark/bucketmapjoin8.q.out +++ b/ql/src/test/results/clientpositive/spark/bucketmapjoin8.q.out @@ -153,7 +153,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 344 Data size: 1377 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -222,7 +222,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 344 Data size: 1377 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -458,7 +458,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 344 Data size: 1377 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -527,7 +527,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 344 Data size: 1377 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/bucketmapjoin9.q.out b/ql/src/test/results/clientpositive/spark/bucketmapjoin9.q.out index d80bdcf..b04f691 100644 --- a/ql/src/test/results/clientpositive/spark/bucketmapjoin9.q.out +++ b/ql/src/test/results/clientpositive/spark/bucketmapjoin9.q.out @@ -161,7 +161,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 525 Data size: 2100 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -230,7 +230,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 344 Data size: 1377 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -499,7 +499,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 344 Data size: 1377 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -568,7 +568,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 344 Data size: 1377 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/bucketmapjoin_negative.q.out b/ql/src/test/results/clientpositive/spark/bucketmapjoin_negative.q.out index 85c51ff..ccbc79f 100644 --- a/ql/src/test/results/clientpositive/spark/bucketmapjoin_negative.q.out +++ b/ql/src/test/results/clientpositive/spark/bucketmapjoin_negative.q.out @@ -162,7 +162,7 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 2100 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -232,7 +232,7 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 1375 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/bucketmapjoin_negative2.q.out b/ql/src/test/results/clientpositive/spark/bucketmapjoin_negative2.q.out index 1fbcc48..ab93bf6 100644 --- a/ql/src/test/results/clientpositive/spark/bucketmapjoin_negative2.q.out +++ b/ql/src/test/results/clientpositive/spark/bucketmapjoin_negative2.q.out @@ -164,7 +164,7 @@ STAGE PLANS: Statistics: Num rows: 29 Data size: 3062 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -282,7 +282,7 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 1375 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/bucketmapjoin_negative3.q.out b/ql/src/test/results/clientpositive/spark/bucketmapjoin_negative3.q.out index cec3da5..47d8903 100644 --- a/ql/src/test/results/clientpositive/spark/bucketmapjoin_negative3.q.out +++ b/ql/src/test/results/clientpositive/spark/bucketmapjoin_negative3.q.out @@ -221,7 +221,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string), value (type: string) Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -287,7 +287,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string), value (type: string) Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -457,7 +457,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string), value (type: string) Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -523,7 +523,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string), value (type: string) Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -691,7 +691,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 2200 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: key (type: string), value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -758,7 +758,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 2200 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: key (type: string), value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -928,7 +928,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string), value (type: string) Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -994,7 +994,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string), value (type: string) Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1164,7 +1164,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string), value (type: string) Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1230,7 +1230,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string), value (type: string) Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1400,7 +1400,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string), value (type: string) Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1466,7 +1466,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string), value (type: string) Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1636,7 +1636,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string), value (type: string) Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1702,7 +1702,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string), value (type: string) Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1872,7 +1872,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string), value (type: string) Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1938,7 +1938,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string), value (type: string) Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -2108,7 +2108,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string), value (type: string) Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -2174,7 +2174,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string), value (type: string) Statistics: Num rows: 6 Data size: 1200 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/ctas.q.out b/ql/src/test/results/clientpositive/spark/ctas.q.out index 0ded266..47bddb3 100644 --- a/ql/src/test/results/clientpositive/spark/ctas.q.out +++ b/ql/src/test/results/clientpositive/spark/ctas.q.out @@ -756,7 +756,7 @@ STAGE PLANS: sort order: ++ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE tag: -1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/groupby_complex_types_multi_single_reducer.q.out b/ql/src/test/results/clientpositive/spark/groupby_complex_types_multi_single_reducer.q.out index 7470843..de1d375 100644 --- a/ql/src/test/results/clientpositive/spark/groupby_complex_types_multi_single_reducer.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby_complex_types_multi_single_reducer.q.out @@ -213,16 +213,16 @@ POSTHOOK: query: SELECT DEST1.* FROM DEST1 POSTHOOK: type: QUERY POSTHOOK: Input: default@dest1 #### A masked pattern was here #### -["105"] 1 -["145"] 1 -["181"] 1 -["195"] 2 -["244"] 1 -["335"] 1 -["336"] 1 -["4"] 1 -["421"] 1 -["83"] 2 +["229"] 2 +["242"] 2 +["30"] 1 +["302"] 1 +["315"] 1 +["333"] 2 +["378"] 1 +["41"] 1 +["468"] 4 +["8"] 1 PREHOOK: query: SELECT DEST2.* FROM DEST2 PREHOOK: type: QUERY PREHOOK: Input: default@dest2 @@ -231,13 +231,13 @@ POSTHOOK: query: SELECT DEST2.* FROM DEST2 POSTHOOK: type: QUERY POSTHOOK: Input: default@dest2 #### A masked pattern was here #### -{"156":"val_156"} 1 -{"170":"val_170"} 1 -{"186":"val_186"} 1 -{"274":"val_274"} 1 -{"332":"val_332"} 1 -{"402":"val_402"} 1 -{"480":"val_480"} 3 -{"5":"val_5"} 3 -{"67":"val_67"} 2 -{"96":"val_96"} 1 +{"120":"val_120"} 2 +{"129":"val_129"} 2 +{"160":"val_160"} 1 +{"26":"val_26"} 2 +{"27":"val_27"} 1 +{"288":"val_288"} 2 +{"298":"val_298"} 3 +{"30":"val_30"} 1 +{"311":"val_311"} 3 +{"74":"val_74"} 1 diff --git a/ql/src/test/results/clientpositive/spark/groupby_ppr.q.out b/ql/src/test/results/clientpositive/spark/groupby_ppr.q.out index 860aa58..1322fa5 100644 --- a/ql/src/test/results/clientpositive/spark/groupby_ppr.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby_ppr.q.out @@ -120,7 +120,7 @@ STAGE PLANS: Map-reduce partition columns: substr(key, 1, 1) (type: string) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE tag: -1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out b/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out index 61dd2be..82f4ad5 100644 --- a/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out @@ -330,7 +330,7 @@ STAGE PLANS: Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: -1 value expressions: _col2 (type: bigint) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1237,7 +1237,7 @@ STAGE PLANS: Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: -1 value expressions: _col3 (type: bigint) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1469,7 +1469,7 @@ STAGE PLANS: Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: -1 value expressions: _col2 (type: bigint) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1747,7 +1747,7 @@ STAGE PLANS: Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: -1 value expressions: _col1 (type: bigint) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -2499,7 +2499,7 @@ STAGE PLANS: Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: -1 value expressions: _col1 (type: bigint) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -2818,7 +2818,7 @@ STAGE PLANS: Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: 0 value expressions: _col1 (type: bigint) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -2903,7 +2903,7 @@ STAGE PLANS: Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: 1 value expressions: _col1 (type: bigint) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -3203,7 +3203,7 @@ STAGE PLANS: Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: 0 value expressions: _col1 (type: bigint) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -3284,7 +3284,7 @@ STAGE PLANS: Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: -1 value expressions: _col2 (type: bigint) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -3394,7 +3394,7 @@ STAGE PLANS: Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: 1 value expressions: _col1 (type: string), _col2 (type: bigint) - auto parallelism: true + auto parallelism: false Stage: Stage-0 Fetch Operator @@ -3497,7 +3497,7 @@ STAGE PLANS: Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: -1 value expressions: _col1 (type: bigint) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out b/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out index 99da734..bbfe916 100644 --- a/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out @@ -331,7 +331,7 @@ STAGE PLANS: Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: -1 value expressions: _col2 (type: bigint) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -401,7 +401,7 @@ STAGE PLANS: Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: -1 value expressions: _col2 (type: bigint) - auto parallelism: true + auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: @@ -1256,7 +1256,7 @@ STAGE PLANS: Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: -1 value expressions: _col3 (type: bigint) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1326,7 +1326,7 @@ STAGE PLANS: Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: -1 value expressions: _col3 (type: bigint) - auto parallelism: true + auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: @@ -1506,7 +1506,7 @@ STAGE PLANS: Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: -1 value expressions: _col2 (type: bigint) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1576,7 +1576,7 @@ STAGE PLANS: Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: -1 value expressions: _col2 (type: bigint) - auto parallelism: true + auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: @@ -1802,7 +1802,7 @@ STAGE PLANS: Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: -1 value expressions: _col1 (type: bigint) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1872,7 +1872,7 @@ STAGE PLANS: Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: -1 value expressions: _col1 (type: bigint) - auto parallelism: true + auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: @@ -2572,7 +2572,7 @@ STAGE PLANS: Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: -1 value expressions: _col1 (type: bigint) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -2642,7 +2642,7 @@ STAGE PLANS: Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: -1 value expressions: _col1 (type: bigint) - auto parallelism: true + auto parallelism: false Reducer 5 Needs Tagging: false Reduce Operator Tree: @@ -2908,7 +2908,7 @@ STAGE PLANS: Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: 0 value expressions: _col1 (type: bigint) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -2993,7 +2993,7 @@ STAGE PLANS: Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: 1 value expressions: _col1 (type: bigint) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -3294,7 +3294,7 @@ STAGE PLANS: Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: 0 value expressions: _col1 (type: bigint) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -3375,7 +3375,7 @@ STAGE PLANS: Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: -1 value expressions: _col2 (type: bigint) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -3481,7 +3481,7 @@ STAGE PLANS: Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: -1 value expressions: _col2 (type: bigint) - auto parallelism: true + auto parallelism: false Reducer 5 Needs Tagging: false Reduce Operator Tree: @@ -3502,7 +3502,7 @@ STAGE PLANS: Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: 1 value expressions: _col1 (type: string), _col2 (type: bigint) - auto parallelism: true + auto parallelism: false Stage: Stage-0 Fetch Operator @@ -3606,7 +3606,7 @@ STAGE PLANS: Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: -1 value expressions: _col1 (type: bigint) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -3676,7 +3676,7 @@ STAGE PLANS: Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: -1 value expressions: _col1 (type: bigint) - auto parallelism: true + auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: diff --git a/ql/src/test/results/clientpositive/spark/input_part2.q.out b/ql/src/test/results/clientpositive/spark/input_part2.q.out index b9e85fe..936982f 100644 --- a/ql/src/test/results/clientpositive/spark/input_part2.q.out +++ b/ql/src/test/results/clientpositive/spark/input_part2.q.out @@ -151,7 +151,7 @@ STAGE PLANS: predicate: ((key < 100) and (ds = '2008-04-08')) (type: boolean) Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), hr (type: string), '2008-04-08' (type: string) + expressions: UDFToInteger(key) (type: int), value (type: string), '12' (type: string), '2008-04-08' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -185,7 +185,7 @@ STAGE PLANS: predicate: ((key < 100) and (ds = '2008-04-09')) (type: boolean) Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), hr (type: string), '2008-04-09' (type: string) + expressions: UDFToInteger(key) (type: int), value (type: string), '12' (type: string), '2008-04-09' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/spark/join11.q.out b/ql/src/test/results/clientpositive/spark/join11.q.out index 673a15d..f982de4 100644 --- a/ql/src/test/results/clientpositive/spark/join11.q.out +++ b/ql/src/test/results/clientpositive/spark/join11.q.out @@ -31,17 +31,17 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 100) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 100) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Map 3 Map Operator Tree: @@ -49,17 +49,17 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 100) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 100) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -69,14 +69,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {VALUE._col0} outputColumnNames: _col0, _col3 - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/spark/join12.q.out b/ql/src/test/results/clientpositive/spark/join12.q.out index 13eb3aa..a6988b7 100644 --- a/ql/src/test/results/clientpositive/spark/join12.q.out +++ b/ql/src/test/results/clientpositive/spark/join12.q.out @@ -37,17 +37,17 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key < 100) and key is not null) and (key < 80)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: (((key < 100) and (key < 80)) and key is not null) (type: boolean) + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Map 3 Map Operator Tree: @@ -55,34 +55,34 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key < 80) and key is not null) and (key < 100)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 80) and (key < 100)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key < 100) and key is not null) and (key < 80)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 100) and (key < 80)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -94,14 +94,14 @@ STAGE PLANS: 1 {VALUE._col0} 2 outputColumnNames: _col0, _col3 - Statistics: Num rows: 59 Data size: 629 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 59 Data size: 629 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 59 Data size: 629 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/spark/join13.q.out b/ql/src/test/results/clientpositive/spark/join13.q.out index 5296870..52b7a54 100644 --- a/ql/src/test/results/clientpositive/spark/join13.q.out +++ b/ql/src/test/results/clientpositive/spark/join13.q.out @@ -38,17 +38,17 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 100) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 100) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Map 4 Map Operator Tree: @@ -56,17 +56,17 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 100) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 100) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Map 5 Map Operator Tree: TableScan @@ -93,15 +93,15 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col2, _col3 - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_col0 + _col2) is not null (type: boolean) - Statistics: Num rows: 46 Data size: 489 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: (_col0 + _col2) (type: double) sort order: + Map-reduce partition columns: (_col0 + _col2) (type: double) - Statistics: Num rows: 46 Data size: 489 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col3 (type: string) Reducer 3 Reduce Operator Tree: @@ -112,14 +112,14 @@ STAGE PLANS: 0 {VALUE._col0} {VALUE._col3} 1 outputColumnNames: _col0, _col3 - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 100 Data size: 1065 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 100 Data size: 1065 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 100 Data size: 1065 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/spark/join14.q.out b/ql/src/test/results/clientpositive/spark/join14.q.out index f44c5dd..dddc98b 100644 --- a/ql/src/test/results/clientpositive/spark/join14.q.out +++ b/ql/src/test/results/clientpositive/spark/join14.q.out @@ -51,13 +51,13 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > 100) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key > 100) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator diff --git a/ql/src/test/results/clientpositive/spark/join16.q.out b/ql/src/test/results/clientpositive/spark/join16.q.out index fde2d48..8b6a7e6 100644 --- a/ql/src/test/results/clientpositive/spark/join16.q.out +++ b/ql/src/test/results/clientpositive/spark/join16.q.out @@ -19,24 +19,24 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((key > 10) and (key > 20)) and key is not null) and value is not null) and (value < 200)) (type: boolean) - Statistics: Num rows: 4 Data size: 42 Basic stats: COMPLETE Column stats: NONE + predicate: ((((key > 10) and value is not null) and (key > 20)) and (value < 200)) (type: boolean) + Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 4 Data size: 42 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE Map 3 Map Operator Tree: TableScan alias: tab Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((key > 20) and key is not null) and value is not null) and (value < 200)) (type: boolean) + predicate: ((((key > 20) and value is not null) and key is not null) and (value < 200)) (type: boolean) Statistics: Num rows: 14 Data size: 148 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string), value (type: string) diff --git a/ql/src/test/results/clientpositive/spark/join17.q.out b/ql/src/test/results/clientpositive/spark/join17.q.out index 6d68ed3..b8d565e 100644 --- a/ql/src/test/results/clientpositive/spark/join17.q.out +++ b/ql/src/test/results/clientpositive/spark/join17.q.out @@ -82,7 +82,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -149,7 +149,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/join26.q.out b/ql/src/test/results/clientpositive/spark/join26.q.out index 3b010a0..a0b153f 100644 --- a/ql/src/test/results/clientpositive/spark/join26.q.out +++ b/ql/src/test/results/clientpositive/spark/join26.q.out @@ -131,7 +131,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE tag: 2 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -200,7 +200,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -266,7 +266,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/join32.q.out b/ql/src/test/results/clientpositive/spark/join32.q.out index 5aea3f3..77c951b 100644 --- a/ql/src/test/results/clientpositive/spark/join32.q.out +++ b/ql/src/test/results/clientpositive/spark/join32.q.out @@ -129,7 +129,7 @@ STAGE PLANS: Map-reduce partition columns: value (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -198,7 +198,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -265,7 +265,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -374,7 +374,7 @@ STAGE PLANS: Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: _col0 (type: string), _col6 (type: string) - auto parallelism: true + auto parallelism: false Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/spark/join32_lessSize.q.out b/ql/src/test/results/clientpositive/spark/join32_lessSize.q.out index bffd620..8b03385 100644 --- a/ql/src/test/results/clientpositive/spark/join32_lessSize.q.out +++ b/ql/src/test/results/clientpositive/spark/join32_lessSize.q.out @@ -137,7 +137,7 @@ STAGE PLANS: Map-reduce partition columns: value (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -206,7 +206,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -273,7 +273,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -382,7 +382,7 @@ STAGE PLANS: Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: _col0 (type: string), _col6 (type: string) - auto parallelism: true + auto parallelism: false Stage: Stage-2 Dependency Collection @@ -647,7 +647,7 @@ STAGE PLANS: Map-reduce partition columns: value (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -714,7 +714,7 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE tag: 2 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -781,7 +781,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -848,7 +848,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: key (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -915,7 +915,7 @@ STAGE PLANS: Map-reduce partition columns: _col5 (type: string) Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Reducer 3 Needs Tagging: true Reduce Operator Tree: @@ -1248,7 +1248,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1315,7 +1315,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1381,7 +1381,7 @@ STAGE PLANS: Map-reduce partition columns: value (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1455,7 +1455,7 @@ STAGE PLANS: Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: _col0 (type: string) - auto parallelism: true + auto parallelism: false Reducer 3 Needs Tagging: true Reduce Operator Tree: @@ -1774,7 +1774,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1841,7 +1841,7 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1907,7 +1907,7 @@ STAGE PLANS: Map-reduce partition columns: value (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1981,7 +1981,7 @@ STAGE PLANS: Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: _col0 (type: string) - auto parallelism: true + auto parallelism: false Reducer 3 Needs Tagging: true Reduce Operator Tree: diff --git a/ql/src/test/results/clientpositive/spark/join33.q.out b/ql/src/test/results/clientpositive/spark/join33.q.out index 5aea3f3..77c951b 100644 --- a/ql/src/test/results/clientpositive/spark/join33.q.out +++ b/ql/src/test/results/clientpositive/spark/join33.q.out @@ -129,7 +129,7 @@ STAGE PLANS: Map-reduce partition columns: value (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -198,7 +198,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -265,7 +265,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -374,7 +374,7 @@ STAGE PLANS: Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: _col0 (type: string), _col6 (type: string) - auto parallelism: true + auto parallelism: false Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/spark/join34.q.out b/ql/src/test/results/clientpositive/spark/join34.q.out index 533c285..29e6734 100644 --- a/ql/src/test/results/clientpositive/spark/join34.q.out +++ b/ql/src/test/results/clientpositive/spark/join34.q.out @@ -159,7 +159,7 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: ((key < 20) and key is not null) (type: boolean) + predicate: (key < 20) (type: boolean) Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 @@ -169,7 +169,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) tag: 0 value expressions: _col1 (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -226,7 +226,7 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: ((key > 100) and key is not null) (type: boolean) + predicate: (key > 100) (type: boolean) Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 @@ -236,7 +236,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) tag: 0 value expressions: _col1 (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -303,7 +303,7 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -363,17 +363,17 @@ STAGE PLANS: 0 {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col1, _col2, _col3 - Statistics: Num rows: 182 Data size: 1938 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 365 Data size: 3878 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col3 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 182 Data size: 1938 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 365 Data size: 3878 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 182 Data size: 1938 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 365 Data size: 3878 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git a/ql/src/test/results/clientpositive/spark/join35.q.out b/ql/src/test/results/clientpositive/spark/join35.q.out index 1750aec..d59c253 100644 --- a/ql/src/test/results/clientpositive/spark/join35.q.out +++ b/ql/src/test/results/clientpositive/spark/join35.q.out @@ -170,26 +170,26 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: ((key < 20) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 20) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: key - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: bigint) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -247,26 +247,26 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: ((key > 100) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key > 100) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: key - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: bigint) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -333,7 +333,7 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -400,7 +400,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) tag: 0 value expressions: _col1 (type: bigint) - auto parallelism: true + auto parallelism: false Reducer 4 Needs Tagging: true Reduce Operator Tree: @@ -411,17 +411,17 @@ STAGE PLANS: 0 {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col1, _col2, _col3 - Statistics: Num rows: 90 Data size: 957 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1938 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col3 (type: string), UDFToInteger(_col1) (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 90 Data size: 957 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1938 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 90 Data size: 957 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1938 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -459,7 +459,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) tag: 0 value expressions: _col1 (type: bigint) - auto parallelism: true + auto parallelism: false Union 3 Vertex: Union 3 diff --git a/ql/src/test/results/clientpositive/spark/join38.q.out b/ql/src/test/results/clientpositive/spark/join38.q.out index cef8a84..220cdfd 100644 --- a/ql/src/test/results/clientpositive/spark/join38.q.out +++ b/ql/src/test/results/clientpositive/spark/join38.q.out @@ -67,7 +67,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 1 Data size: 128 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (col11 is not null and (col11 = 111)) (type: boolean) + predicate: (col11 = 111) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: '111' (type: string) @@ -80,12 +80,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key = 111)) (type: boolean) - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 111) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: '111' (type: string) sort order: + - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Reducer 2 Reduce Operator Tree: @@ -96,22 +96,22 @@ STAGE PLANS: 0 {VALUE._col0} 1 {VALUE._col5} outputColumnNames: _col1, _col10 - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col10 (type: string) outputColumnNames: _col1, _col10 - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col1 (type: string), _col10 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Reducer 3 Reduce Operator Tree: @@ -120,14 +120,14 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 68 Data size: 724 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 68 Data size: 724 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 68 Data size: 724 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/spark/join9.q.out b/ql/src/test/results/clientpositive/spark/join9.q.out index 5553ad2..f5ad508 100644 --- a/ql/src/test/results/clientpositive/spark/join9.q.out +++ b/ql/src/test/results/clientpositive/spark/join9.q.out @@ -98,7 +98,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -164,7 +164,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/join_filters_overlap.q.out b/ql/src/test/results/clientpositive/spark/join_filters_overlap.q.out index 78c2b69..09002f9 100644 --- a/ql/src/test/results/clientpositive/spark/join_filters_overlap.q.out +++ b/ql/src/test/results/clientpositive/spark/join_filters_overlap.q.out @@ -121,7 +121,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: int) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -188,7 +188,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE tag: 2 value expressions: value (type: int) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -251,7 +251,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 21 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: int) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -476,7 +476,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 21 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: int) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -543,7 +543,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE tag: 2 value expressions: value (type: int) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -610,7 +610,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: int) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -849,7 +849,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 21 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: int) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -916,7 +916,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE tag: 2 value expressions: value (type: int) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -983,7 +983,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: int) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1240,7 +1240,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE tag: 3 value expressions: value (type: int) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1303,7 +1303,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 21 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: int) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1370,7 +1370,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE tag: 2 value expressions: value (type: int) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1433,7 +1433,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 21 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: int) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1685,7 +1685,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE tag: 3 value expressions: value (type: int) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1752,7 +1752,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: int) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1819,7 +1819,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE tag: 2 value expressions: value (type: int) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1882,7 +1882,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 21 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: int) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/join_map_ppr.q.out b/ql/src/test/results/clientpositive/spark/join_map_ppr.q.out index 4ee6b8d..78d81f1 100644 --- a/ql/src/test/results/clientpositive/spark/join_map_ppr.q.out +++ b/ql/src/test/results/clientpositive/spark/join_map_ppr.q.out @@ -133,7 +133,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE tag: 2 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -202,7 +202,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -268,7 +268,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -693,7 +693,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE tag: 2 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -762,7 +762,7 @@ STAGE PLANS: Statistics: Num rows: 28 Data size: 2958 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -829,7 +829,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: key (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/join_vc.q.out b/ql/src/test/results/clientpositive/spark/join_vc.q.out index 6e34ef3..6a6fad2 100644 --- a/ql/src/test/results/clientpositive/spark/join_vc.q.out +++ b/ql/src/test/results/clientpositive/spark/join_vc.q.out @@ -127,3 +127,235 @@ POSTHOOK: Input: default@src 0 238 val_238 0 238 val_238 0 238 val_238 +PREHOOK: query: explain +select t2.BLOCK__OFFSET__INSIDE__FILE +from src t1 join src t2 on t1.key = t2.key where t1.key < 100 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select t2.BLOCK__OFFSET__INSIDE__FILE +from src t1 join src t2 on t1.key = t2.key where t1.key < 100 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1), Map 3 (GROUP PARTITION-LEVEL SORT, 1) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key < 100) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + value expressions: BLOCK__OFFSET__INSIDE__FILE (type: bigint) + Map 3 + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key < 100) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {VALUE._col1} + outputColumnNames: _col7 + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col7 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select t2.BLOCK__OFFSET__INSIDE__FILE +from src t1 join src t2 on t1.key = t2.key where t1.key < 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select t2.BLOCK__OFFSET__INSIDE__FILE +from src t1 join src t2 on t1.key = t2.key where t1.key < 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +968 +2088 +2632 +968 +2088 +2632 +968 +2088 +2632 +2846 +3170 +1720 +4362 +1720 +4362 +386 +2770 +386 +2770 +910 +5340 +5514 +5340 +5514 +2824 +4004 +1118 +1972 +4594 +1972 +4594 +2226 +5284 +2226 +5284 +34 +5616 +3494 +3592 +3192 +1238 +3138 +4012 +1238 +3138 +4012 +1238 +3138 +4012 +328 +5626 +328 +5626 +1218 +3388 +2030 +3298 +2030 +3298 +2330 +4068 +1198 +3060 +3864 +4540 +3060 +3864 +4540 +3060 +3864 +4540 +1462 +2308 +1462 +2308 +4186 +1440 +1024 +1906 +3128 +1906 +3128 +3516 +1592 +198 +1754 +5306 +1754 +5306 +3570 +3794 +4548 +4640 +3794 +4548 +4640 +3794 +4548 +4640 +1208 +2792 +1208 +2792 +3548 +3378 +3538 +3378 +3538 +2622 +3368 +1916 +4058 +396 +1674 +5070 +1674 +5070 +1872 +5606 +1872 +5606 +2612 +12 +2652 +5398 +2802 +4304 +5744 +2802 +4304 +5744 +2802 +4304 +5744 +1176 +2400 +3160 +2400 +3160 +2216 +5572 +5802 +5572 +5802 +92 +2458 +92 +2458 diff --git a/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out b/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out index 94b38f7..520c2aa 100644 --- a/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out +++ b/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out @@ -1028,25 +1028,25 @@ POSTHOOK: query: select value, sum(key) as sum from src group by value limit 20 POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### -val_0 0.0 +val_10 10.0 val_100 200.0 val_103 206.0 -val_104 208.0 val_105 105.0 val_11 11.0 val_111 111.0 -val_113 226.0 val_114 114.0 +val_116 116.0 val_118 236.0 val_119 357.0 val_12 24.0 val_120 240.0 val_125 250.0 -val_126 126.0 -val_129 258.0 +val_128 384.0 val_131 131.0 +val_137 137.0 val_138 414.0 val_15 15.0 +val_174 174.0 val_193 193.0 PREHOOK: query: -- flush for order-by explain @@ -1291,8 +1291,6 @@ POSTHOOK: Input: default@src 11.0 111.0 114.0 -1150.0 -116.0 126.0 131.0 133.0 @@ -1304,15 +1302,19 @@ POSTHOOK: Input: default@src 155.0 156.0 157.0 +158.0 +160.0 162.0 163.0 166.0 -168.0 17.0 +170.0 177.0 178.0 180.0 181.0 +183.0 +186.0 189.0 19.0 192.0 @@ -1321,67 +1323,65 @@ POSTHOOK: Input: default@src 20.0 200.0 201.0 +202.0 206.0 -208.0 214.0 -218.0 +222.0 226.0 226.0 228.0 -235.0 -236.0 24.0 24.0 240.0 242.0 250.0 -255.0 +258.0 268.0 272.0 273.0 274.0 -278.0 +280.0 282.0 -292.0 298.0 -30.0 304.0 +309.0 316.0 327.0 328.0 330.0 -344.0 +348.0 348.0 350.0 352.0 357.0 +358.0 36.0 382.0 384.0 390.0 394.0 +399.0 400.0 406.0 -410.0 417.0 418.0 426.0 430.0 432.0 -434.0 -438.0 442.0 446.0 +448.0 +458.0 459.0 466.0 +469.0 474.0 478.0 489.0 -501.0 552.0 -554.0 +561.0 579.0 597.0 622.0 624.0 -676.0 +636.0 diff --git a/ql/src/test/results/clientpositive/spark/louter_join_ppr.q.out b/ql/src/test/results/clientpositive/spark/louter_join_ppr.q.out index d4cd429..fd2a7be 100644 --- a/ql/src/test/results/clientpositive/spark/louter_join_ppr.q.out +++ b/ql/src/test/results/clientpositive/spark/louter_join_ppr.q.out @@ -128,7 +128,7 @@ STAGE PLANS: Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -244,7 +244,7 @@ STAGE PLANS: Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -509,7 +509,7 @@ STAGE PLANS: Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -576,7 +576,7 @@ STAGE PLANS: Statistics: Num rows: 222 Data size: 2358 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: string), ds (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -993,7 +993,7 @@ STAGE PLANS: Statistics: Num rows: 222 Data size: 2358 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string), ds (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1203,7 +1203,7 @@ STAGE PLANS: Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1472,7 +1472,7 @@ STAGE PLANS: Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1539,7 +1539,7 @@ STAGE PLANS: Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/mapjoin_mapjoin.q.out b/ql/src/test/results/clientpositive/spark/mapjoin_mapjoin.q.out index 1801d13..e71b0ca 100644 --- a/ql/src/test/results/clientpositive/spark/mapjoin_mapjoin.q.out +++ b/ql/src/test/results/clientpositive/spark/mapjoin_mapjoin.q.out @@ -80,7 +80,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: key (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -289,7 +289,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -355,7 +355,7 @@ STAGE PLANS: Map-reduce partition columns: value (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -422,7 +422,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Reducer 3 Needs Tagging: true Reduce Operator Tree: @@ -517,13 +517,13 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (value is not null and (value > 'val_450')) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (value > 'val_450') (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: value (type: string) sort order: + Map-reduce partition columns: value (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator diff --git a/ql/src/test/results/clientpositive/spark/optimize_nullscan.q.out b/ql/src/test/results/clientpositive/spark/optimize_nullscan.q.out index d9de8d9..d715bf0 100644 --- a/ql/src/test/results/clientpositive/spark/optimize_nullscan.q.out +++ b/ql/src/test/results/clientpositive/spark/optimize_nullscan.q.out @@ -245,7 +245,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: -mr-10002default.src{} [src] Path -> Partition: @@ -561,7 +561,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Stage: Stage-0 Fetch Operator @@ -1540,75 +1540,48 @@ STAGE PLANS: #### A masked pattern was here #### Vertices: Map 1 + Map 4 Map Operator Tree: TableScan - alias: srcpart - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + alias: src GatherStats: false - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - tag: 1 - auto parallelism: true - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: hr=11 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - hr 11 - properties: - COLUMN_STATS_ACCURATE true - bucket_count -1 - columns key,value - columns.comments defaultdefault - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - numFiles 1 - numRows 500 - partition_columns ds/hr - partition_columns.types string:string - rawDataSize 5312 - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 + Filter Operator + isSamplingPred: false + predicate: false (type: boolean) + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns key,value - columns.comments defaultdefault - columns.types string:string + NumFilesPerFileSink: 1 #### A masked pattern was here #### - name default.srcpart - partition_columns ds/hr - partition_columns.types string:string - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0 + columns.types string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Path -> Alias: #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcpart - name: default.srcpart + Path -> Partition: #### A masked pattern was here #### Partition - base file name: hr=12 + base file name: src input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - hr 12 properties: COLUMN_STATS_ACCURATE true bucket_count -1 @@ -1616,13 +1589,11 @@ STAGE PLANS: columns.comments defaultdefault columns.types string:string #### A masked pattern was here #### - name default.srcpart + name default.src numFiles 1 numRows 500 - partition_columns ds/hr - partition_columns.types string:string rawDataSize 5312 - serialization.ddl struct srcpart { string key, string value} + serialization.ddl struct src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 5812 @@ -1632,29 +1603,51 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: + COLUMN_STATS_ACCURATE true bucket_count -1 columns key,value columns.comments defaultdefault columns.types string:string #### A masked pattern was here #### - name default.srcpart - partition_columns ds/hr - partition_columns.types string:string - serialization.ddl struct srcpart { string key, string value} + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcpart - name: default.srcpart -#### A masked pattern was here #### + name: default.src + name: default.src + Truncated Path -> Alias: + /src [src] + Map 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: false (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + tag: 0 + auto parallelism: false + Path -> Alias: + -mr-10002default.src{} [src] + Path -> Partition: + -mr-10002default.src{} Partition - base file name: hr=11 - input format: org.apache.hadoop.mapred.TextInputFormat + base file name: src + input format: org.apache.hadoop.hive.ql.io.OneNullRowInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-09 - hr 11 properties: COLUMN_STATS_ACCURATE true bucket_count -1 @@ -1662,129 +1655,171 @@ STAGE PLANS: columns.comments defaultdefault columns.types string:string #### A masked pattern was here #### - name default.srcpart + name default.src numFiles 1 numRows 500 - partition_columns ds/hr - partition_columns.types string:string rawDataSize 5312 - serialization.ddl struct srcpart { string key, string value} + serialization.ddl struct src { string key, string value} serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serialization.lib org.apache.hadoop.hive.serde2.NullStructSerDe totalSize 5812 #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.NullStructSerDe input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: + COLUMN_STATS_ACCURATE true bucket_count -1 columns key,value columns.comments defaultdefault columns.types string:string #### A masked pattern was here #### - name default.srcpart - partition_columns ds/hr - partition_columns.types string:string - serialization.ddl struct srcpart { string key, string value} + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcpart - name: default.srcpart + name: default.src + name: default.src + Truncated Path -> Alias: + -mr-10002default.src{} [src] + Reducer 2 + Needs Tagging: true + Reduce Operator Tree: + Join Operator + condition map: + Left Outer Join0 to 1 + condition expressions: + 0 {KEY.reducesinkkey0} + 1 + outputColumnNames: _col0 + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 #### A masked pattern was here #### - Partition - base file name: hr=12 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-09 - hr 12 - properties: - COLUMN_STATS_ACCURATE true - bucket_count -1 - columns key,value - columns.comments defaultdefault - columns.types string:string + NumFilesPerFileSink: 1 #### A masked pattern was here #### - name default.srcpart - numFiles 1 - numRows 500 - partition_columns ds/hr - partition_columns.types string:string - rawDataSize 5312 - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0 + columns.types string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Union 3 + Vertex: Union 3 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * from (select key from src union all select src.key from src left outer join srcpart on src.key = srcpart.key) a where false +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@srcpart #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns key,value - columns.comments defaultdefault - columns.types string:string +POSTHOOK: query: select * from (select key from src union all select src.key from src left outer join srcpart on src.key = srcpart.key) a where false +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@srcpart #### A masked pattern was here #### - name default.srcpart - partition_columns ds/hr - partition_columns.types string:string - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +PREHOOK: query: explain extended +select * from src s1, src s2 where false and s1.value = s2.value +PREHOOK: type: QUERY +POSTHOOK: query: explain extended +select * from src s1, src s2 where false and s1.value = s2.value +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_JOIN + TOK_TABREF + TOK_TABNAME + src + s1 + TOK_TABREF + TOK_TABNAME + src + s2 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_WHERE + and + false + = + . + TOK_TABLE_OR_COL + s1 + value + . + TOK_TABLE_OR_COL + s2 + value + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1), Map 3 (GROUP PARTITION-LEVEL SORT, 1) #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcpart - name: default.srcpart - Truncated Path -> Alias: - /srcpart/ds=2008-04-08/hr=11 [srcpart] - /srcpart/ds=2008-04-08/hr=12 [srcpart] - /srcpart/ds=2008-04-09/hr=11 [srcpart] - /srcpart/ds=2008-04-09/hr=12 [srcpart] - Map 4 + Vertices: + Map 1 Map Operator Tree: TableScan - alias: src + alias: s2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: false (type: boolean) - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - columns _col0 - columns.types string - escape.delim \ - hive.serialization.extend.nesting.levels true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + tag: 1 + value expressions: key (type: string) + auto parallelism: false Path -> Alias: -#### A masked pattern was here #### + -mr-10002default.src{} [s2] Path -> Partition: -#### A masked pattern was here #### + -mr-10002default.src{} Partition base file name: src - input format: org.apache.hadoop.mapred.TextInputFormat + input format: org.apache.hadoop.hive.ql.io.OneNullRowInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: COLUMN_STATS_ACCURATE true @@ -1799,10 +1834,10 @@ STAGE PLANS: rawDataSize 5312 serialization.ddl struct src { string key, string value} serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serialization.lib org.apache.hadoop.hive.serde2.NullStructSerDe totalSize 5812 #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.NullStructSerDe input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1826,27 +1861,32 @@ STAGE PLANS: name: default.src name: default.src Truncated Path -> Alias: - /src [src] - Map 5 + -mr-10002default.src{} [s2] + Map 3 Map Operator Tree: TableScan - alias: src + alias: s1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - tag: 0 - auto parallelism: true + Filter Operator + isSamplingPred: false + predicate: false (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + tag: 0 + value expressions: key (type: string) + auto parallelism: false Path -> Alias: -#### A masked pattern was here #### + -mr-10003default.src{} [s1] Path -> Partition: -#### A masked pattern was here #### + -mr-10003default.src{} Partition base file name: src - input format: org.apache.hadoop.mapred.TextInputFormat + input format: org.apache.hadoop.hive.ql.io.OneNullRowInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: COLUMN_STATS_ACCURATE true @@ -1861,10 +1901,10 @@ STAGE PLANS: rawDataSize 5312 serialization.ddl struct src { string key, string value} serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serialization.lib org.apache.hadoop.hive.serde2.NullStructSerDe totalSize 5812 #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.NullStructSerDe input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1888,48 +1928,43 @@ STAGE PLANS: name: default.src name: default.src Truncated Path -> Alias: - /src [src] + -mr-10003default.src{} [s1] Reducer 2 Needs Tagging: true Reduce Operator Tree: Join Operator condition map: - Left Outer Join0 to 1 + Inner Join 0 to 1 condition expressions: - 0 {KEY.reducesinkkey0} - 1 - outputColumnNames: _col0 - Filter Operator - isSamplingPred: false - predicate: false (type: boolean) - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - columns _col0 - columns.types string - escape.delim \ - hive.serialization.extend.nesting.levels true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Union 3 - Vertex: Union 3 + 0 {VALUE._col0} {KEY.reducesinkkey0} + 1 {VALUE._col0} {KEY.reducesinkkey0} + outputColumnNames: _col0, _col1, _col5, _col6 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types string:string:string:string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -1937,21 +1972,11 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: select * from (select key from src union all select src.key from src left outer join srcpart on src.key = srcpart.key) a where false +PREHOOK: query: select * from src s1, src s2 where false and s1.value = s2.value PREHOOK: type: QUERY PREHOOK: Input: default@src -PREHOOK: Input: default@srcpart -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### -POSTHOOK: query: select * from (select key from src union all select src.key from src left outer join srcpart on src.key = srcpart.key) a where false +POSTHOOK: query: select * from src s1, src s2 where false and s1.value = s2.value POSTHOOK: type: QUERY POSTHOOK: Input: default@src -POSTHOOK: Input: default@srcpart -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/outer_join_ppr.q.out b/ql/src/test/results/clientpositive/spark/outer_join_ppr.q.out index 6e44a91..2645554 100644 --- a/ql/src/test/results/clientpositive/spark/outer_join_ppr.q.out +++ b/ql/src/test/results/clientpositive/spark/outer_join_ppr.q.out @@ -124,7 +124,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string), ds (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -330,7 +330,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -600,7 +600,7 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string), ds (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -806,7 +806,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/pcr.q.out b/ql/src/test/results/clientpositive/spark/pcr.q.out index 4e9244f..f12787b 100644 --- a/ql/src/test/results/clientpositive/spark/pcr.q.out +++ b/ql/src/test/results/clientpositive/spark/pcr.q.out @@ -2776,8 +2776,8 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: NONE tag: 1 - value expressions: value (type: string), ds (type: string) - auto parallelism: true + value expressions: value (type: string) + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -2844,8 +2844,8 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: NONE tag: 0 - value expressions: value (type: string), ds (type: string) - auto parallelism: true + value expressions: value (type: string) + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -2903,26 +2903,26 @@ STAGE PLANS: condition map: Inner Join 0 to 1 condition expressions: - 0 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} - 1 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} - outputColumnNames: _col0, _col1, _col2, _col6, _col7, _col8 + 0 {KEY.reducesinkkey0} {VALUE._col0} + 1 {KEY.reducesinkkey0} {VALUE._col0} + outputColumnNames: _col0, _col1, _col6, _col7 Statistics: Num rows: 1 Data size: 198 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col6 (type: int), _col7 (type: string), _col8 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + expressions: _col0 (type: int), _col1 (type: string), _col6 (type: int), _col7 (type: string) + outputColumnNames: _col0, _col1, _col3, _col4 Statistics: Num rows: 1 Data size: 198 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Statistics: Num rows: 1 Data size: 198 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: int), _col4 (type: string), _col5 (type: string) + value expressions: _col1 (type: string), _col3 (type: int), _col4 (type: string) auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: string), VALUE._col4 (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), '2000-04-08' (type: string), VALUE._col2 (type: int), VALUE._col3 (type: string), '2000-04-08' (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: 198 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -3084,8 +3084,8 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: NONE tag: 1 - value expressions: value (type: string), ds (type: string) - auto parallelism: true + value expressions: value (type: string) + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -3152,8 +3152,8 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: NONE tag: 0 - value expressions: value (type: string), ds (type: string) - auto parallelism: true + value expressions: value (type: string) + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -3211,26 +3211,26 @@ STAGE PLANS: condition map: Inner Join 0 to 1 condition expressions: - 0 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} - 1 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} - outputColumnNames: _col0, _col1, _col2, _col6, _col7, _col8 + 0 {KEY.reducesinkkey0} {VALUE._col0} + 1 {KEY.reducesinkkey0} {VALUE._col0} + outputColumnNames: _col0, _col1, _col6, _col7 Statistics: Num rows: 1 Data size: 198 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col6 (type: int), _col7 (type: string), _col8 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + expressions: _col0 (type: int), _col1 (type: string), _col6 (type: int), _col7 (type: string) + outputColumnNames: _col0, _col1, _col3, _col4 Statistics: Num rows: 1 Data size: 198 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Statistics: Num rows: 1 Data size: 198 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: int), _col4 (type: string), _col5 (type: string) + value expressions: _col1 (type: string), _col3 (type: int), _col4 (type: string) auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: string), VALUE._col4 (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), '2000-04-08' (type: string), VALUE._col2 (type: int), VALUE._col3 (type: string), '2000-04-09' (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: 198 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -4837,11 +4837,11 @@ STAGE PLANS: predicate: (key = 11) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: value (type: string), ds (type: string), hr (type: string) - outputColumnNames: _col1, _col2, _col3 + expressions: value (type: string), hr (type: string) + outputColumnNames: _col1, _col3 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: '11' (type: string), _col2 (type: string), _col3 (type: string) + key expressions: '11' (type: string), '2008-04-08' (type: string), _col3 (type: string) sort order: +++ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE tag: -1 @@ -4949,7 +4949,7 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: '11' (type: string), VALUE._col0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string) + expressions: '11' (type: string), VALUE._col0 (type: string), '2008-04-08' (type: string), KEY.reducesinkkey2 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -5066,11 +5066,11 @@ STAGE PLANS: predicate: (key = 11) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: value (type: string), ds (type: string), hr (type: string) - outputColumnNames: _col1, _col2, _col3 + expressions: value (type: string), ds (type: string) + outputColumnNames: _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: '11' (type: string), _col2 (type: string), _col3 (type: string) + key expressions: '11' (type: string), _col2 (type: string), '11' (type: string) sort order: +++ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE tag: -1 @@ -5178,7 +5178,7 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: '11' (type: string), VALUE._col0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string) + expressions: '11' (type: string), VALUE._col0 (type: string), KEY.reducesinkkey1 (type: string), '11' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -5468,3 +5468,66 @@ POSTHOOK: query: drop table foo_field POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@foo_field POSTHOOK: Output: default@foo_field +PREHOOK: query: explain select key,value from srcpart where cast(hr as double) = cast(11 as double) +PREHOOK: type: QUERY +POSTHOOK: query: explain select key,value from srcpart where cast(hr as double) = cast(11 as double) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: srcpart + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + ListSink + +PREHOOK: query: explain select key,value from srcpart where hr = cast(11 as double) +PREHOOK: type: QUERY +POSTHOOK: query: explain select key,value from srcpart where hr = cast(11 as double) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: srcpart + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + ListSink + +PREHOOK: query: explain select key,value from srcpart where cast(hr as double) = 11 +PREHOOK: type: QUERY +POSTHOOK: query: explain select key,value from srcpart where cast(hr as double) = 11 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: srcpart + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + ListSink + diff --git a/ql/src/test/results/clientpositive/spark/ppd_gby_join.q.out b/ql/src/test/results/clientpositive/spark/ppd_gby_join.q.out index c5c34c1..1b6acee 100644 --- a/ql/src/test/results/clientpositive/spark/ppd_gby_join.q.out +++ b/ql/src/test/results/clientpositive/spark/ppd_gby_join.q.out @@ -36,40 +36,40 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((key > '2') and (key < '400')) and key is not null) and (key <> '4')) and (key > '20')) (type: boolean) - Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE + predicate: ((((key > '2') and (key < '400')) and (key <> '4')) and (key > '20')) (type: boolean) + Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col0 < '400') and _col0 is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 21 Basic stats: COMPLETE Column stats: NONE + predicate: (_col0 < '400') (type: boolean) + Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((((key > '1') and (key < '400')) and key is not null) and (key > '20')) and ((value < 'val_50') or (key > '2'))) and (key <> '4')) (type: boolean) - Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE + predicate: (((((key > '1') and (key < '400')) and (key > '20')) and ((value < 'val_50') or (key > '2'))) and (key <> '4')) (type: boolean) + Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col0 < '400') and _col0 is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + predicate: (_col0 < '400') (type: boolean) + Statistics: Num rows: 4 Data size: 42 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 42 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reducer 2 Reduce Operator Tree: @@ -80,7 +80,7 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 23 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 69 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((((_col0 > '20') and ((_col1 < 'val_50') or (_col0 > '2'))) and ((_col2 > '50') or (_col0 < '50'))) and (_col2 <> '4')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE @@ -164,34 +164,34 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((key > '2') and (key < '400')) and key is not null) and (key <> '4')) and (key > '20')) (type: boolean) - Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE + predicate: ((((key > '2') and (key < '400')) and (key <> '4')) and (key > '20')) (type: boolean) + Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((((key > '1') and (key < '400')) and key is not null) and (key > '20')) and ((value < 'val_50') or (key > '2'))) and (key <> '4')) (type: boolean) - Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE + predicate: (((((key > '1') and (key < '400')) and (key > '20')) and ((value < 'val_50') or (key > '2'))) and (key <> '4')) (type: boolean) + Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reducer 2 Reduce Operator Tree: @@ -202,25 +202,25 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 9 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 19 Data size: 210 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((((_col0 > '20') and ((_col1 < 'val_50') or (_col0 > '2'))) and ((_col2 > '50') or (_col0 < '50'))) and (_col2 <> '4')) (type: boolean) - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reducer 3 Reduce Operator Tree: @@ -229,14 +229,14 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/spark/ppd_join.q.out b/ql/src/test/results/clientpositive/spark/ppd_join.q.out index 48ed98b..34be421 100644 --- a/ql/src/test/results/clientpositive/spark/ppd_join.q.out +++ b/ql/src/test/results/clientpositive/spark/ppd_join.q.out @@ -33,20 +33,20 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((key > '2') and (key < '400')) and key is not null) and (key <> '4')) and (key > '20')) (type: boolean) - Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE + predicate: ((((key > '2') and (key < '400')) and (key <> '4')) and (key > '20')) (type: boolean) + Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col0 < '400') and _col0 is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 21 Basic stats: COMPLETE Column stats: NONE + predicate: (_col0 < '400') (type: boolean) + Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 21 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Map 3 Map Operator Tree: @@ -54,20 +54,20 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((((key > '1') and (key < '400')) and key is not null) and (key > '20')) and ((value < 'val_50') or (key > '2'))) and (key <> '4')) (type: boolean) - Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE + predicate: (((((key > '1') and (key < '400')) and (key > '20')) and ((value < 'val_50') or (key > '2'))) and (key <> '4')) (type: boolean) + Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col0 < '400') and _col0 is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + predicate: (_col0 < '400') (type: boolean) + Statistics: Num rows: 4 Data size: 42 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 42 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reducer 2 Reduce Operator Tree: @@ -78,7 +78,7 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2 Data size: 23 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 69 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((((_col0 > '20') and ((_col1 < 'val_50') or (_col0 > '2'))) and ((_col2 > '50') or (_col0 < '50'))) and (_col2 <> '4')) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE @@ -573,17 +573,17 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((key > '2') and (key < '400')) and key is not null) and (key <> '4')) and (key > '20')) (type: boolean) - Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE + predicate: ((((key > '2') and (key < '400')) and (key <> '4')) and (key > '20')) (type: boolean) + Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Map 3 Map Operator Tree: @@ -591,17 +591,17 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((((key > '1') and (key < '400')) and key is not null) and (key > '20')) and ((value < 'val_50') or (key > '2'))) and (key <> '4')) (type: boolean) - Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE + predicate: (((((key > '1') and (key < '400')) and (key > '20')) and ((value < 'val_50') or (key > '2'))) and (key <> '4')) (type: boolean) + Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reducer 2 Reduce Operator Tree: @@ -612,17 +612,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 9 Data size: 104 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 19 Data size: 210 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((((_col0 > '20') and ((_col1 < 'val_50') or (_col0 > '2'))) and ((_col2 > '50') or (_col0 < '50'))) and (_col2 <> '4')) (type: boolean) - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/spark/ppd_join2.q.out b/ql/src/test/results/clientpositive/spark/ppd_join2.q.out index 21228c1..d5e2b14 100644 --- a/ql/src/test/results/clientpositive/spark/ppd_join2.q.out +++ b/ql/src/test/results/clientpositive/spark/ppd_join2.q.out @@ -40,20 +40,20 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((key <> '305') and (key < '400')) and key is not null) and (key <> '14')) and (key <> '311')) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: ((((key <> '305') and (key < '400')) and (key <> '14')) and (key <> '311')) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col0 < '400') and _col0 is not null) (type: boolean) - Statistics: Num rows: 14 Data size: 148 Basic stats: COMPLETE Column stats: NONE + predicate: (_col0 < '400') (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 14 Data size: 148 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Map 4 Map Operator Tree: @@ -61,20 +61,20 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((((key <> '302') and (key < '400')) and key is not null) and value is not null) and (key <> '311')) and ((value <> 'val_50') or (key > '1'))) and (key <> '14')) (type: boolean) - Statistics: Num rows: 56 Data size: 594 Basic stats: COMPLETE Column stats: NONE + predicate: ((((((key <> '302') and (key < '400')) and value is not null) and (key <> '311')) and ((value <> 'val_50') or (key > '1'))) and (key <> '14')) (type: boolean) + Statistics: Num rows: 110 Data size: 1168 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 56 Data size: 594 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 110 Data size: 1168 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col0 < '400') and _col0 is not null) (type: boolean) - Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE + predicate: (_col0 < '400') (type: boolean) + Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Map 5 Map Operator Tree: @@ -106,15 +106,15 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 15 Data size: 162 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 60 Data size: 642 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col1 is not null (type: boolean) - Statistics: Num rows: 8 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 30 Data size: 321 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 8 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 30 Data size: 321 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col2 (type: string), _col3 (type: string) Reducer 3 Reduce Operator Tree: @@ -1733,17 +1733,17 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((key <> '305') and (key < '400')) and key is not null) and (key <> '14')) and (key <> '311')) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: ((((key <> '305') and (key < '400')) and (key <> '14')) and (key <> '311')) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Map 4 Map Operator Tree: @@ -1751,17 +1751,17 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((((key <> '302') and (key < '400')) and key is not null) and value is not null) and (key <> '311')) and ((value <> 'val_50') or (key > '1'))) and (key <> '14')) (type: boolean) - Statistics: Num rows: 56 Data size: 594 Basic stats: COMPLETE Column stats: NONE + predicate: ((((((key <> '302') and (key < '400')) and value is not null) and (key <> '311')) and ((value <> 'val_50') or (key > '1'))) and (key <> '14')) (type: boolean) + Statistics: Num rows: 110 Data size: 1168 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 56 Data size: 594 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 110 Data size: 1168 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 56 Data size: 594 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 110 Data size: 1168 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Map 5 Map Operator Tree: @@ -1790,12 +1790,12 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col2 (type: string), _col3 (type: string) Reducer 3 Reduce Operator Tree: diff --git a/ql/src/test/results/clientpositive/spark/ppd_join3.q.out b/ql/src/test/results/clientpositive/spark/ppd_join3.q.out index f95f53e..b899967 100644 --- a/ql/src/test/results/clientpositive/spark/ppd_join3.q.out +++ b/ql/src/test/results/clientpositive/spark/ppd_join3.q.out @@ -39,20 +39,20 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((((key <> '12') and (key < '400')) and key is not null) and (key <> '4')) and (key > '0')) and (key <> '1')) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: (((((key <> '12') and (key < '400')) and (key <> '4')) and (key > '0')) and (key <> '1')) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col0 < '400') and _col0 is not null) (type: boolean) - Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + predicate: (_col0 < '400') (type: boolean) + Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Map 3 Map Operator Tree: @@ -60,20 +60,20 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((((key <> '11') and (key < '400')) and key is not null) and (key > '0')) and ((value <> 'val_500') or (key > '1'))) and (key <> '4')) and (key <> '1')) (type: boolean) - Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE + predicate: ((((((key <> '11') and (key < '400')) and (key > '0')) and ((value <> 'val_500') or (key > '1'))) and (key <> '4')) and (key <> '1')) (type: boolean) + Statistics: Num rows: 73 Data size: 775 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 73 Data size: 775 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col0 < '400') and _col0 is not null) (type: boolean) - Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE + predicate: (_col0 < '400') (type: boolean) + Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Map 4 Map Operator Tree: @@ -81,20 +81,20 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((((key <> '13') and (key < '400')) and key is not null) and (key <> '1')) and (key > '0')) and (key <> '4')) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: (((((key <> '13') and (key < '400')) and (key <> '1')) and (key > '0')) and (key <> '4')) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col0 < '400') and _col0 is not null) (type: boolean) - Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + predicate: (_col0 < '400') (type: boolean) + Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 5 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -106,17 +106,17 @@ STAGE PLANS: 1 {KEY.reducesinkkey0} {VALUE._col0} 2 {KEY.reducesinkkey0} outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 52 Data size: 558 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((((_col0 > '0') and ((_col1 <> 'val_500') or (_col0 > '1'))) and ((_col2 > '10') or (_col0 <> '10'))) and (_col2 <> '4')) and (_col4 <> '1')) (type: boolean) - Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 311 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 311 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 311 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1770,17 +1770,17 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((((key <> '12') and (key < '400')) and key is not null) and (key <> '4')) and (key > '0')) and (key <> '1')) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: (((((key <> '12') and (key < '400')) and (key <> '4')) and (key > '0')) and (key <> '1')) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Map 3 Map Operator Tree: @@ -1788,17 +1788,17 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((((key <> '11') and (key < '400')) and key is not null) and (key > '0')) and ((value <> 'val_500') or (key > '1'))) and (key <> '4')) and (key <> '1')) (type: boolean) - Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE + predicate: ((((((key <> '11') and (key < '400')) and (key > '0')) and ((value <> 'val_500') or (key > '1'))) and (key <> '4')) and (key <> '1')) (type: boolean) + Statistics: Num rows: 73 Data size: 775 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 73 Data size: 775 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 73 Data size: 775 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Map 4 Map Operator Tree: @@ -1806,17 +1806,17 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((((key <> '13') and (key < '400')) and key is not null) and (key <> '1')) and (key > '0')) and (key <> '4')) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: (((((key <> '13') and (key < '400')) and (key <> '1')) and (key > '0')) and (key <> '4')) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -1828,17 +1828,17 @@ STAGE PLANS: 1 {KEY.reducesinkkey0} {VALUE._col0} 2 {KEY.reducesinkkey0} outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 79 Data size: 840 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 160 Data size: 1705 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((((_col0 > '0') and ((_col1 <> 'val_500') or (_col0 > '1'))) and ((_col2 > '10') or (_col0 <> '10'))) and (_col2 <> '4')) and (_col4 <> '1')) (type: boolean) - Statistics: Num rows: 45 Data size: 478 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 93 Data size: 991 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 45 Data size: 478 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 93 Data size: 991 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 45 Data size: 478 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 93 Data size: 991 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/spark/ppd_join4.q.out b/ql/src/test/results/clientpositive/spark/ppd_join4.q.out index 36b5ed5..8af4001 100644 --- a/ql/src/test/results/clientpositive/spark/ppd_join4.q.out +++ b/ql/src/test/results/clientpositive/spark/ppd_join4.q.out @@ -63,7 +63,7 @@ STAGE PLANS: alias: t3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Filter Operator - predicate: (id is not null and (id = 'a')) (type: boolean) + predicate: (id = 'a') (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator key expressions: 'a' (type: string) diff --git a/ql/src/test/results/clientpositive/spark/ppd_join_filter.q.out b/ql/src/test/results/clientpositive/spark/ppd_join_filter.q.out index c6cb1b4..66c616d 100644 --- a/ql/src/test/results/clientpositive/spark/ppd_join_filter.q.out +++ b/ql/src/test/results/clientpositive/spark/ppd_join_filter.q.out @@ -147,7 +147,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -224,7 +224,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -334,7 +334,7 @@ STAGE PLANS: Statistics: Num rows: 21 Data size: 223 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: _col3 (type: double), _col4 (type: double) - auto parallelism: true + auto parallelism: false Stage: Stage-0 Fetch Operator @@ -525,7 +525,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -602,7 +602,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -712,7 +712,7 @@ STAGE PLANS: Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: _col3 (type: double), _col4 (type: double) - auto parallelism: true + auto parallelism: false Stage: Stage-0 Fetch Operator @@ -903,7 +903,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -976,7 +976,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1086,7 +1086,7 @@ STAGE PLANS: Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: _col3 (type: double), _col4 (type: double) - auto parallelism: true + auto parallelism: false Stage: Stage-0 Fetch Operator @@ -1277,7 +1277,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1354,7 +1354,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1464,7 +1464,7 @@ STAGE PLANS: Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: _col3 (type: double), _col4 (type: double) - auto parallelism: true + auto parallelism: false Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/spark/ptf_matchpath.q.out b/ql/src/test/results/clientpositive/spark/ptf_matchpath.q.out index f04c998..fc0187b 100644 --- a/ql/src/test/results/clientpositive/spark/ptf_matchpath.q.out +++ b/ql/src/test/results/clientpositive/spark/ptf_matchpath.q.out @@ -107,3 +107,27 @@ Baltimore 1142 2010 10 21 5 21 Baltimore 1142 2010 10 22 4 22 Baltimore 1142 2010 10 25 3 25 Baltimore 1142 2010 10 26 2 26 +PREHOOK: query: -- 3. empty partition. +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + (select * from flights_tiny where fl_num = -1142) flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +POSTHOOK: query: -- 3. empty partition. +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + (select * from flights_tiny where fl_num = -1142) flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny +#### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/router_join_ppr.q.out b/ql/src/test/results/clientpositive/spark/router_join_ppr.q.out index 8389a81..84171c2 100644 --- a/ql/src/test/results/clientpositive/spark/router_join_ppr.q.out +++ b/ql/src/test/results/clientpositive/spark/router_join_ppr.q.out @@ -128,7 +128,7 @@ STAGE PLANS: Statistics: Num rows: 222 Data size: 2358 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string), ds (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -338,7 +338,7 @@ STAGE PLANS: Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -612,7 +612,7 @@ STAGE PLANS: Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -679,7 +679,7 @@ STAGE PLANS: Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -993,7 +993,7 @@ STAGE PLANS: Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1109,7 +1109,7 @@ STAGE PLANS: Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1374,7 +1374,7 @@ STAGE PLANS: Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1441,7 +1441,7 @@ STAGE PLANS: Statistics: Num rows: 222 Data size: 2358 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: string), ds (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/sample1.q.out b/ql/src/test/results/clientpositive/spark/sample1.q.out index e6c5e62..0296589 100644 --- a/ql/src/test/results/clientpositive/spark/sample1.q.out +++ b/ql/src/test/results/clientpositive/spark/sample1.q.out @@ -79,7 +79,7 @@ STAGE PLANS: predicate: (((hash(rand()) & 2147483647) % 1) = 0) (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), ds (type: string), hr (type: string) + expressions: UDFToInteger(key) (type: int), value (type: string), '2008-04-08' (type: string), '11' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/spark/sample10.q.out b/ql/src/test/results/clientpositive/spark/sample10.q.out index c511152..339f80c 100644 --- a/ql/src/test/results/clientpositive/spark/sample10.q.out +++ b/ql/src/test/results/clientpositive/spark/sample10.q.out @@ -121,7 +121,7 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 702 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: bigint) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/sample8.q.out b/ql/src/test/results/clientpositive/spark/sample8.q.out index 6296900..3ac7356 100644 --- a/ql/src/test/results/clientpositive/spark/sample8.q.out +++ b/ql/src/test/results/clientpositive/spark/sample8.q.out @@ -103,15 +103,15 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: true - predicate: ((((((hash(key) & 2147483647) % 1) = 0) and key is not null) and value is not null) and (((hash(key) & 2147483647) % 10) = 0)) (type: boolean) - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + predicate: (((((hash(key) & 2147483647) % 1) = 0) and value is not null) and (((hash(key) & 2147483647) % 10) = 0)) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string), value (type: string) sort order: ++ Map-reduce partition columns: key (type: string), value (type: string) - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -312,16 +312,15 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: true - predicate: ((((((hash(key) & 2147483647) % 10) = 0) and key is not null) and value is not null) and (((hash(key) & 2147483647) % 1) = 0)) (type: boolean) - Statistics: Num rows: 31 Data size: 329 Basic stats: COMPLETE Column stats: NONE + predicate: (((((hash(key) & 2147483647) % 10) = 0) and value is not null) and (((hash(key) & 2147483647) % 1) = 0)) (type: boolean) + Statistics: Num rows: 62 Data size: 658 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string), value (type: string) sort order: ++ Map-reduce partition columns: key (type: string), value (type: string) - Statistics: Num rows: 31 Data size: 329 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 62 Data size: 658 Basic stats: COMPLETE Column stats: NONE tag: 0 - value expressions: ds (type: string), hr (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -380,24 +379,24 @@ STAGE PLANS: condition map: Inner Join 0 to 1 condition expressions: - 0 {KEY.reducesinkkey0} {KEY.reducesinkkey1} {VALUE._col0} {VALUE._col1} + 0 {KEY.reducesinkkey0} {KEY.reducesinkkey1} 1 {KEY.reducesinkkey0} {KEY.reducesinkkey1} - outputColumnNames: _col0, _col1, _col2, _col3, _col7, _col8 - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col7, _col8 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false - predicate: ((((_col7 = _col0) and (_col8 = _col1)) and (_col2 = '2008-04-08')) and (_col3 = '11')) (type: boolean) - Statistics: Num rows: 8 Data size: 85 Basic stats: COMPLETE Column stats: NONE + predicate: ((_col7 = _col0) and (_col8 = _col1)) (type: boolean) + Statistics: Num rows: 68 Data size: 722 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), '11' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 8 Data size: 85 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 68 Data size: 722 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 8 Data size: 85 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 68 Data size: 722 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git a/ql/src/test/results/clientpositive/spark/semijoin.q.out b/ql/src/test/results/clientpositive/spark/semijoin.q.out index 18fc837..584285a 100644 --- a/ql/src/test/results/clientpositive/spark/semijoin.q.out +++ b/ql/src/test/results/clientpositive/spark/semijoin.q.out @@ -453,22 +453,22 @@ STAGE PLANS: alias: b Statistics: Num rows: 46 Data size: 185 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 15) and key is not null) (type: boolean) - Statistics: Num rows: 8 Data size: 32 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 15) (type: boolean) + Statistics: Num rows: 15 Data size: 60 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: key - Statistics: Num rows: 8 Data size: 32 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 60 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: key (type: int), key (type: int) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 8 Data size: 32 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 60 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: int) sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 8 Data size: 32 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 60 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan @@ -492,24 +492,24 @@ STAGE PLANS: 0 {VALUE._col0} 1 outputColumnNames: _col1 - Statistics: Num rows: 8 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 66 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 8 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 66 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + - Statistics: Num rows: 8 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 66 Basic stats: COMPLETE Column stats: NONE Reducer 3 Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 8 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 66 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 8 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 66 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -681,22 +681,22 @@ STAGE PLANS: alias: t3 Statistics: Num rows: 46 Data size: 185 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > 5) and key is not null) (type: boolean) - Statistics: Num rows: 8 Data size: 32 Basic stats: COMPLETE Column stats: NONE + predicate: (key > 5) (type: boolean) + Statistics: Num rows: 15 Data size: 60 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 8 Data size: 32 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 60 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: int) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 8 Data size: 32 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 60 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 8 Data size: 32 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 60 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -706,24 +706,24 @@ STAGE PLANS: 0 {VALUE._col0} 1 outputColumnNames: _col1 - Statistics: Num rows: 8 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 66 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 8 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 66 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + - Statistics: Num rows: 8 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 66 Basic stats: COMPLETE Column stats: NONE Reducer 3 Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 8 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 66 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 8 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 66 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -870,22 +870,22 @@ STAGE PLANS: alias: t1 Statistics: Num rows: 22 Data size: 90 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > 2) and key is not null) (type: boolean) - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE + predicate: (key > 2) (type: boolean) + Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: int) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan @@ -909,24 +909,24 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Reducer 3 Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/spark/skewjoin.q.out b/ql/src/test/results/clientpositive/spark/skewjoin.q.out index d674d04..3be091e 100644 --- a/ql/src/test/results/clientpositive/spark/skewjoin.q.out +++ b/ql/src/test/results/clientpositive/spark/skewjoin.q.out @@ -842,17 +842,17 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key < 100) and key is not null) and (key < 80)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: (((key < 100) and (key < 80)) and key is not null) (type: boolean) + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Map 4 Map Operator Tree: @@ -860,34 +860,34 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key < 80) and key is not null) and (key < 100)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 80) and (key < 100)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Map 5 Map Operator Tree: TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key < 100) and key is not null) and (key < 80)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < 100) and (key < 80)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -899,11 +899,11 @@ STAGE PLANS: 1 {VALUE._col0} 2 outputColumnNames: _col0, _col3 - Statistics: Num rows: 59 Data size: 629 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col3 (type: string) outputColumnNames: _col0, _col3 - Statistics: Num rows: 59 Data size: 629 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(hash(_col0)), sum(hash(_col3)) mode: hash diff --git a/ql/src/test/results/clientpositive/spark/skewjoinopt10.q.out b/ql/src/test/results/clientpositive/spark/skewjoinopt10.q.out index 9e85b11..f11f11e 100644 --- a/ql/src/test/results/clientpositive/spark/skewjoinopt10.q.out +++ b/ql/src/test/results/clientpositive/spark/skewjoinopt10.q.out @@ -133,6 +133,8 @@ STAGE PLANS: Lateral View Forward Select Operator SELECT * : (no compute) + expressions: _col0 (type: string), _col1 (type: array) + outputColumnNames: org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc, org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc Lateral View Join Operator outputColumnNames: _col0, _col1, _col2 Select Operator @@ -177,6 +179,8 @@ STAGE PLANS: Lateral View Forward Select Operator SELECT * : (no compute) + expressions: _col0 (type: string), _col1 (type: array) + outputColumnNames: org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc, org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc Lateral View Join Operator outputColumnNames: _col0, _col1, _col2 Select Operator diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin9.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin9.q.out index 43ea4cd..403abcb 100644 --- a/ql/src/test/results/clientpositive/spark/smb_mapjoin9.q.out +++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin9.q.out @@ -401,7 +401,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: int) Statistics: Num rows: 28 Data size: 2958 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string), ds (type: string) + value expressions: value (type: string) Map 3 Map Operator Tree: TableScan @@ -422,11 +422,11 @@ STAGE PLANS: Inner Join 0 to 1 condition expressions: 0 {KEY.reducesinkkey0} - 1 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} - outputColumnNames: _col0, _col6, _col7, _col8 + 1 {KEY.reducesinkkey0} {VALUE._col0} + outputColumnNames: _col0, _col6, _col7 Statistics: Num rows: 799 Data size: 3198 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col6 (type: int), _col7 (type: string), _col8 (type: string), _col0 (type: int) + expressions: _col6 (type: int), _col7 (type: string), '2010-10-15' (type: string), _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 799 Data size: 3198 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_10.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_10.q.out index 341a8c3..747150b 100644 --- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_10.q.out +++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_10.q.out @@ -96,7 +96,6 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: userid (type: int), pageid (type: int), postid (type: int), type (type: string) Statistics: Num rows: 1 Data size: 138 Basic stats: COMPLETE Column stats: NONE - value expressions: ds (type: string) Map 3 Map Operator Tree: TableScan @@ -110,19 +109,18 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: userid (type: int), pageid (type: int), postid (type: int), type (type: string) Statistics: Num rows: 1 Data size: 138 Basic stats: COMPLETE Column stats: NONE - value expressions: ds (type: string) Reducer 2 Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 condition expressions: - 0 {KEY.reducesinkkey0} {KEY.reducesinkkey1} {KEY.reducesinkkey2} {KEY.reducesinkkey3} {VALUE._col0} - 1 {KEY.reducesinkkey0} {KEY.reducesinkkey1} {KEY.reducesinkkey2} {KEY.reducesinkkey3} {VALUE._col0} - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col8, _col9, _col10, _col11, _col12 + 0 {KEY.reducesinkkey0} {KEY.reducesinkkey1} {KEY.reducesinkkey2} {KEY.reducesinkkey3} + 1 {KEY.reducesinkkey0} {KEY.reducesinkkey1} {KEY.reducesinkkey2} {KEY.reducesinkkey3} + outputColumnNames: _col0, _col1, _col2, _col3, _col8, _col9, _col10, _col11 Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: string), _col4 (type: string), _col8 (type: int), _col9 (type: int), _col10 (type: int), _col11 (type: string), _col12 (type: string) + expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: string), '1' (type: string), _col8 (type: int), _col9 (type: int), _col10 (type: int), _col11 (type: string), '2' (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_13.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_13.q.out index 0319137..f65b0ef 100644 --- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_13.q.out +++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_13.q.out @@ -153,7 +153,7 @@ STAGE PLANS: Statistics: Num rows: 28 Data size: 2958 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: key (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -224,7 +224,7 @@ STAGE PLANS: Statistics: Num rows: 28 Data size: 2958 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -441,7 +441,7 @@ STAGE PLANS: Statistics: Num rows: 28 Data size: 2958 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: key (type: int), value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -512,7 +512,7 @@ STAGE PLANS: Statistics: Num rows: 28 Data size: 2958 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: key (type: int), value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out index cad4063..db562f1 100644 --- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out +++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out @@ -490,34 +490,34 @@ STAGE PLANS: alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -526,9 +526,9 @@ STAGE PLANS: condition expressions: 0 1 - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -938,34 +938,34 @@ STAGE PLANS: alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 8) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 8) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 8) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 8) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -974,9 +974,9 @@ STAGE PLANS: condition expressions: 0 1 - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 3 Data size: 13 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -1190,17 +1190,17 @@ STAGE PLANS: alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan @@ -1306,17 +1306,17 @@ STAGE PLANS: alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan @@ -1432,51 +1432,51 @@ STAGE PLANS: alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Map 5 Map Operator Tree: TableScan alias: a Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -1487,9 +1487,9 @@ STAGE PLANS: 0 1 2 - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_15.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_15.q.out index 7849e78..ad70142 100644 --- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_15.q.out +++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_15.q.out @@ -123,7 +123,7 @@ STAGE PLANS: Statistics: Num rows: 28 Data size: 2958 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -194,7 +194,7 @@ STAGE PLANS: Statistics: Num rows: 28 Data size: 2958 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -475,7 +475,7 @@ STAGE PLANS: Statistics: Num rows: 18 Data size: 1956 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -546,7 +546,7 @@ STAGE PLANS: Statistics: Num rows: 18 Data size: 1956 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -773,7 +773,7 @@ STAGE PLANS: Statistics: Num rows: 18 Data size: 1956 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -844,7 +844,7 @@ STAGE PLANS: Statistics: Num rows: 18 Data size: 1956 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: value (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1071,7 +1071,7 @@ STAGE PLANS: Statistics: Num rows: 18 Data size: 1956 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: key2 (type: int) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1142,7 +1142,7 @@ STAGE PLANS: Statistics: Num rows: 18 Data size: 1956 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: key2 (type: int) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_25.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_25.q.out index efa38d4..131f72d 100644 --- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_25.q.out +++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_25.q.out @@ -71,48 +71,48 @@ STAGE PLANS: alias: b Statistics: Num rows: 51 Data size: 206 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key = 5)) (type: boolean) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 5) (type: boolean) + Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: 5 (type: int) sort order: + - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan alias: a Statistics: Num rows: 52 Data size: 208 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key = 5)) (type: boolean) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 5) (type: boolean) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: 5 (type: int) sort order: + - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: NONE Map 5 Map Operator Tree: TableScan alias: d Statistics: Num rows: 55 Data size: 222 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key = 5)) (type: boolean) - Statistics: Num rows: 14 Data size: 56 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 5) (type: boolean) + Statistics: Num rows: 27 Data size: 108 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: 5 (type: int) sort order: + - Statistics: Num rows: 14 Data size: 56 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 108 Basic stats: COMPLETE Column stats: NONE Map 7 Map Operator Tree: TableScan alias: c Statistics: Num rows: 51 Data size: 206 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key = 5)) (type: boolean) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 5) (type: boolean) + Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: 5 (type: int) sort order: + - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -121,16 +121,16 @@ STAGE PLANS: condition expressions: 0 1 - Statistics: Num rows: 14 Data size: 57 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 114 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 5 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 14 Data size: 57 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 114 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 14 Data size: 57 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 114 Basic stats: COMPLETE Column stats: NONE Reducer 3 Reduce Operator Tree: Join Operator @@ -140,17 +140,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {KEY.reducesinkkey0} outputColumnNames: _col0, _col1 - Statistics: Num rows: 16 Data size: 67 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 129 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_col1 = 5) (type: boolean) - Statistics: Num rows: 8 Data size: 33 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 62 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), 5 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 8 Data size: 33 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 62 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 8 Data size: 33 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 62 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -163,16 +163,16 @@ STAGE PLANS: condition expressions: 0 1 - Statistics: Num rows: 15 Data size: 61 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 118 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 5 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 15 Data size: 61 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 118 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 15 Data size: 61 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 118 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator @@ -215,48 +215,48 @@ STAGE PLANS: alias: b Statistics: Num rows: 51 Data size: 206 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key = 5)) (type: boolean) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 5) (type: boolean) + Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: 5 (type: int) sort order: + - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan alias: a Statistics: Num rows: 52 Data size: 208 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key = 5)) (type: boolean) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 5) (type: boolean) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: 5 (type: int) sort order: + - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: NONE Map 5 Map Operator Tree: TableScan alias: d Statistics: Num rows: 55 Data size: 222 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key = 5)) (type: boolean) - Statistics: Num rows: 14 Data size: 56 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 5) (type: boolean) + Statistics: Num rows: 27 Data size: 108 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: 5 (type: int) sort order: + - Statistics: Num rows: 14 Data size: 56 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 108 Basic stats: COMPLETE Column stats: NONE Map 7 Map Operator Tree: TableScan alias: c Statistics: Num rows: 51 Data size: 206 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key = 5)) (type: boolean) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 5) (type: boolean) + Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: 5 (type: int) sort order: + - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -265,16 +265,16 @@ STAGE PLANS: condition expressions: 0 1 - Statistics: Num rows: 14 Data size: 57 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 114 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 5 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 14 Data size: 57 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 114 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 14 Data size: 57 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 114 Basic stats: COMPLETE Column stats: NONE Reducer 3 Reduce Operator Tree: Join Operator @@ -284,17 +284,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {KEY.reducesinkkey0} outputColumnNames: _col0, _col1 - Statistics: Num rows: 16 Data size: 67 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 31 Data size: 129 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_col1 = 5) (type: boolean) - Statistics: Num rows: 8 Data size: 33 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 62 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), 5 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 8 Data size: 33 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 62 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 8 Data size: 33 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 62 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -307,16 +307,16 @@ STAGE PLANS: condition expressions: 0 1 - Statistics: Num rows: 15 Data size: 61 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 118 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 5 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 15 Data size: 61 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 118 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 15 Data size: 61 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 118 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_6.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_6.q.out index 1836f12..c814a09 100644 --- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_6.q.out +++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_6.q.out @@ -2448,13 +2448,13 @@ STAGE PLANS: alias: b Statistics: Num rows: 48 Data size: 5008 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key > 1000)) (type: boolean) - Statistics: Num rows: 8 Data size: 834 Basic stats: COMPLETE Column stats: NONE + predicate: (key > 1000) (type: boolean) + Statistics: Num rows: 16 Data size: 1669 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: int) sort order: + Map-reduce partition columns: key (type: int) - Statistics: Num rows: 8 Data size: 834 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1669 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Map 3 Map Operator Tree: @@ -2462,13 +2462,13 @@ STAGE PLANS: alias: a Statistics: Num rows: 48 Data size: 5008 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key > 1000)) (type: boolean) - Statistics: Num rows: 8 Data size: 834 Basic stats: COMPLETE Column stats: NONE + predicate: (key > 1000) (type: boolean) + Statistics: Num rows: 16 Data size: 1669 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: int) sort order: + Map-reduce partition columns: key (type: int) - Statistics: Num rows: 8 Data size: 834 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1669 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Reducer 2 Reduce Operator Tree: @@ -2479,14 +2479,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col5, _col6 - Statistics: Num rows: 8 Data size: 917 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 1835 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 8 Data size: 917 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 1835 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 8 Data size: 917 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 1835 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -2553,13 +2553,13 @@ STAGE PLANS: alias: b Statistics: Num rows: 48 Data size: 5008 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key > 1000)) (type: boolean) - Statistics: Num rows: 8 Data size: 834 Basic stats: COMPLETE Column stats: NONE + predicate: (key > 1000) (type: boolean) + Statistics: Num rows: 16 Data size: 1669 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: int) sort order: + Map-reduce partition columns: key (type: int) - Statistics: Num rows: 8 Data size: 834 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1669 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Map 3 Map Operator Tree: @@ -2567,13 +2567,13 @@ STAGE PLANS: alias: a Statistics: Num rows: 48 Data size: 5008 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key > 1000)) (type: boolean) - Statistics: Num rows: 8 Data size: 834 Basic stats: COMPLETE Column stats: NONE + predicate: (key > 1000) (type: boolean) + Statistics: Num rows: 16 Data size: 1669 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: int) sort order: + Map-reduce partition columns: key (type: int) - Statistics: Num rows: 8 Data size: 834 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1669 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Reducer 2 Reduce Operator Tree: @@ -2584,14 +2584,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col5, _col6 - Statistics: Num rows: 8 Data size: 917 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 1835 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 8 Data size: 917 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 1835 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 8 Data size: 917 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 1835 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -2654,13 +2654,13 @@ STAGE PLANS: alias: b Statistics: Num rows: 48 Data size: 5008 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key > 1000)) (type: boolean) - Statistics: Num rows: 8 Data size: 834 Basic stats: COMPLETE Column stats: NONE + predicate: (key > 1000) (type: boolean) + Statistics: Num rows: 16 Data size: 1669 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: int) sort order: + Map-reduce partition columns: key (type: int) - Statistics: Num rows: 8 Data size: 834 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1669 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Map 3 Map Operator Tree: @@ -2668,13 +2668,13 @@ STAGE PLANS: alias: c Statistics: Num rows: 48 Data size: 5008 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key > 1000)) (type: boolean) - Statistics: Num rows: 8 Data size: 834 Basic stats: COMPLETE Column stats: NONE + predicate: (key > 1000) (type: boolean) + Statistics: Num rows: 16 Data size: 1669 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: int) sort order: + Map-reduce partition columns: key (type: int) - Statistics: Num rows: 8 Data size: 834 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1669 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Map 4 Map Operator Tree: @@ -2682,13 +2682,13 @@ STAGE PLANS: alias: a Statistics: Num rows: 48 Data size: 5008 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key > 1000)) (type: boolean) - Statistics: Num rows: 8 Data size: 834 Basic stats: COMPLETE Column stats: NONE + predicate: (key > 1000) (type: boolean) + Statistics: Num rows: 16 Data size: 1669 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: int) sort order: + Map-reduce partition columns: key (type: int) - Statistics: Num rows: 8 Data size: 834 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 1669 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: string) Reducer 2 Reduce Operator Tree: @@ -2701,14 +2701,14 @@ STAGE PLANS: 1 {KEY.reducesinkkey0} {VALUE._col0} 2 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11 - Statistics: Num rows: 17 Data size: 1834 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 35 Data size: 3671 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col10 (type: int), _col11 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 17 Data size: 1834 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 35 Data size: 3671 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 17 Data size: 1834 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 35 Data size: 3671 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_1.q.out b/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_1.q.out index 32c3818..af32d33 100644 --- a/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_1.q.out +++ b/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_1.q.out @@ -74,26 +74,26 @@ STAGE PLANS: alias: b Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key < 10)) (type: boolean) - Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan alias: a Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key < 10)) (type: boolean) - Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -102,9 +102,9 @@ STAGE PLANS: condition expressions: 0 1 - Statistics: Num rows: 9 Data size: 991 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 2093 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 9 Data size: 991 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 2093 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash diff --git a/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_5.q.out b/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_5.q.out index f59d942..5cf007e 100644 --- a/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_5.q.out +++ b/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_5.q.out @@ -141,7 +141,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 727 Data size: 2908 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -211,7 +211,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 727 Data size: 2908 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_6.q.out b/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_6.q.out index 4085d9a..71333b8 100644 --- a/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_6.q.out +++ b/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_6.q.out @@ -141,7 +141,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 727 Data size: 2908 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -210,7 +210,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 727 Data size: 2908 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_7.q.out b/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_7.q.out index 28336c5..5de9a90 100644 --- a/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_7.q.out +++ b/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_7.q.out @@ -179,7 +179,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 1453 Data size: 5812 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -296,7 +296,7 @@ STAGE PLANS: Map-reduce partition columns: key (type: int) Statistics: Num rows: 1453 Data size: 5812 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_8.q.out b/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_8.q.out index 087a89d..0d22f4c 100644 --- a/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_8.q.out +++ b/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_8.q.out @@ -129,26 +129,26 @@ STAGE PLANS: alias: b Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key < 10)) (type: boolean) - Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan alias: a Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key is not null and (key < 10)) (type: boolean) - Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 9 Data size: 901 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -157,9 +157,9 @@ STAGE PLANS: condition expressions: 0 1 - Statistics: Num rows: 9 Data size: 991 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 2093 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 9 Data size: 991 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 2093 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash diff --git a/ql/src/test/results/clientpositive/spark/subquery_exists.q.out b/ql/src/test/results/clientpositive/spark/subquery_exists.q.out index d5628f6..8244da5 100644 --- a/ql/src/test/results/clientpositive/spark/subquery_exists.q.out +++ b/ql/src/test/results/clientpositive/spark/subquery_exists.q.out @@ -48,22 +48,22 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((value > 'val_9') and value is not null) and key is not null) (type: boolean) - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + predicate: ((value > 'val_9') and key is not null) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: string), key (type: string) outputColumnNames: _col1, _col2 - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col1 (type: string), _col2 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator diff --git a/ql/src/test/results/clientpositive/spark/subquery_in.q.out b/ql/src/test/results/clientpositive/spark/subquery_in.q.out index 323c894..29e2fd6 100644 --- a/ql/src/test/results/clientpositive/spark/subquery_in.q.out +++ b/ql/src/test/results/clientpositive/spark/subquery_in.q.out @@ -127,22 +127,22 @@ STAGE PLANS: alias: s1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > '9') and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key > '9') (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Map 3 Map Operator Tree: TableScan @@ -258,22 +258,22 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key > '9') and key is not null) and value is not null) (type: boolean) - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > '9') and value is not null) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -696,22 +696,22 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key > '9') and key is not null) and value is not null) (type: boolean) - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > '9') and value is not null) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: key, value - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: key (type: string), value (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -739,21 +739,21 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 21 Data size: 223 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 21 Data size: 223 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 21 Data size: 223 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 21 Data size: 223 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/spark/subquery_multiinsert.q.out b/ql/src/test/results/clientpositive/spark/subquery_multiinsert.q.out index 2bedd37..29b6d57 100644 --- a/ql/src/test/results/clientpositive/spark/subquery_multiinsert.q.out +++ b/ql/src/test/results/clientpositive/spark/subquery_multiinsert.q.out @@ -95,22 +95,22 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key > '9') and key is not null) and value is not null) (type: boolean) - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > '9') and value is not null) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Map 7 Map Operator Tree: TableScan @@ -530,22 +530,22 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key > '9') and key is not null) and value is not null) (type: boolean) - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > '9') and value is not null) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Map 7 Map Operator Tree: TableScan diff --git a/ql/src/test/results/clientpositive/spark/transform_ppr1.q.out b/ql/src/test/results/clientpositive/spark/transform_ppr1.q.out index 5309ade..70bf0fd 100644 --- a/ql/src/test/results/clientpositive/spark/transform_ppr1.q.out +++ b/ql/src/test/results/clientpositive/spark/transform_ppr1.q.out @@ -136,7 +136,7 @@ STAGE PLANS: Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: '2008-04-08' (type: string), _col1 (type: string), _col2 (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/transform_ppr2.q.out b/ql/src/test/results/clientpositive/spark/transform_ppr2.q.out index 2dc285f..ea7ec5a 100644 --- a/ql/src/test/results/clientpositive/spark/transform_ppr2.q.out +++ b/ql/src/test/results/clientpositive/spark/transform_ppr2.q.out @@ -111,7 +111,7 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE GatherStats: false Select Operator - expressions: ds (type: string), key (type: string), value (type: string) + expressions: '2008-04-08' (type: string), key (type: string), value (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Transform Operator @@ -138,7 +138,7 @@ STAGE PLANS: Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: diff --git a/ql/src/test/results/clientpositive/spark/union15.q.out b/ql/src/test/results/clientpositive/spark/union15.q.out index 1c35ead..1d5adb3 100644 --- a/ql/src/test/results/clientpositive/spark/union15.q.out +++ b/ql/src/test/results/clientpositive/spark/union15.q.out @@ -116,14 +116,14 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1200 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1200 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 12 Data size: 1200 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -155,20 +155,20 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@src1 #### A masked pattern was here #### -278 2 -273 2 -128 2 255 2 -tst1 1 +213 2 238 2 -146 2 150 2 - 20 -406 2 -369 2 -213 2 311 2 -224 2 +146 2 +369 2 66 2 + 20 +278 2 401 2 +406 2 +tst1 1 +128 2 98 2 +273 2 +224 2 diff --git a/ql/src/test/results/clientpositive/spark/union19.q.out b/ql/src/test/results/clientpositive/spark/union19.q.out index c86afb0..1314523 100644 --- a/ql/src/test/results/clientpositive/spark/union19.q.out +++ b/ql/src/test/results/clientpositive/spark/union19.q.out @@ -134,14 +134,14 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 125 Data size: 12500 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 125 Data size: 12500 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 125 Data size: 12500 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/spark/union29.q.out b/ql/src/test/results/clientpositive/spark/union29.q.out index f82d561..1ea7ecb 100644 --- a/ql/src/test/results/clientpositive/spark/union29.q.out +++ b/ql/src/test/results/clientpositive/spark/union29.q.out @@ -70,18 +70,15 @@ STAGE PLANS: expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Select Operator - expressions: _col0 (type: string), _col1 (type: string) + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) outputColumnNames: _col0, _col1 - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.union_subq_union + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union Map 4 Map Operator Tree: TableScan @@ -90,18 +87,15 @@ STAGE PLANS: expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Select Operator - expressions: _col0 (type: string), _col1 (type: string) + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) outputColumnNames: _col0, _col1 - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.union_subq_union + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union Union 2 Vertex: Union 2 diff --git a/ql/src/test/results/clientpositive/spark/union30.q.out b/ql/src/test/results/clientpositive/spark/union30.q.out index ee0daf4..67a2424 100644 --- a/ql/src/test/results/clientpositive/spark/union30.q.out +++ b/ql/src/test/results/clientpositive/spark/union30.q.out @@ -128,18 +128,15 @@ STAGE PLANS: expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Select Operator - expressions: _col0 (type: string), _col1 (type: string) + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) outputColumnNames: _col0, _col1 - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.union_subq_union + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union Reducer 2 Reduce Operator Tree: Group By Operator @@ -154,18 +151,15 @@ STAGE PLANS: expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 Select Operator - expressions: _col0 (type: string), _col1 (type: string) + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) outputColumnNames: _col0, _col1 - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.union_subq_union + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union Reducer 5 Reduce Operator Tree: Group By Operator @@ -180,18 +174,15 @@ STAGE PLANS: expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 Select Operator - expressions: _col0 (type: string), _col1 (type: string) + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) outputColumnNames: _col0, _col1 - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.union_subq_union + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union Union 3 Vertex: Union 3 diff --git a/ql/src/test/results/clientpositive/spark/union7.q.out b/ql/src/test/results/clientpositive/spark/union7.q.out index b5a693e..c386f6a 100644 --- a/ql/src/test/results/clientpositive/spark/union7.q.out +++ b/ql/src/test/results/clientpositive/spark/union7.q.out @@ -92,14 +92,14 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 600 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 600 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 600 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -127,20 +127,20 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@src1 #### A masked pattern was here #### -278 1 -273 1 -128 1 255 1 -tst1 1 +213 1 238 1 -146 1 150 1 - 10 -406 1 -369 1 -213 1 311 1 -224 1 +146 1 +369 1 66 1 + 10 +278 1 401 1 +406 1 +tst1 1 +128 1 98 1 +273 1 +224 1 diff --git a/ql/src/test/results/clientpositive/spark/union_ppr.q.out b/ql/src/test/results/clientpositive/spark/union_ppr.q.out index 9ed0c86..24b6912 100644 --- a/ql/src/test/results/clientpositive/spark/union_ppr.q.out +++ b/ql/src/test/results/clientpositive/spark/union_ppr.q.out @@ -125,16 +125,16 @@ STAGE PLANS: isSamplingPred: false predicate: (key < 100) (type: boolean) Select Operator - expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + expressions: key (type: string), value (type: string), hr (type: string) + outputColumnNames: _col0, _col1, _col3 Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string) + outputColumnNames: _col0, _col1, _col3 Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + key expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), _col3 (type: string) sort order: ++++ tag: -1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -242,16 +242,16 @@ STAGE PLANS: isSamplingPred: false predicate: (key < 100) (type: boolean) Select Operator - expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + expressions: key (type: string), value (type: string), hr (type: string) + outputColumnNames: _col0, _col1, _col3 Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string) + outputColumnNames: _col0, _col1, _col3 Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + key expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), _col3 (type: string) sort order: ++++ tag: -1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -354,7 +354,7 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string) + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), '2008-04-08' (type: string), KEY.reducesinkkey3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 666 Data size: 7074 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/spark/union_remove_10.q.out b/ql/src/test/results/clientpositive/spark/union_remove_10.q.out index f4601fd..aa49521 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_10.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_10.q.out @@ -142,16 +142,13 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Select Operator - expressions: _col0 (type: string), _col1 (type: bigint) - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1 Reducer 2 Reduce Operator Tree: Group By Operator @@ -165,16 +162,13 @@ STAGE PLANS: Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Select Operator - expressions: _col0 (type: string), _col1 (type: bigint) - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1 Union 3 Vertex: Union 3 @@ -257,7 +251,6 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@inputtbl1 POSTHOOK: Output: default@outputtbl1 POSTHOOK: Lineage: outputtbl1.key EXPRESSION [(inputtbl1)inputtbl1.FieldSchema(name:key, type:string, comment:null), (inputtbl1)inputtbl1.FieldSchema(name:key, type:string, comment:null), (inputtbl1)inputtbl1.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: outputtbl1.values EXPRESSION [(inputtbl1)inputtbl1.null, ] PREHOOK: query: desc formatted outputTbl1 PREHOOK: type: DESCTABLE PREHOOK: Input: default@outputtbl1 diff --git a/ql/src/test/results/clientpositive/spark/union_remove_11.q.out b/ql/src/test/results/clientpositive/spark/union_remove_11.q.out index 21292aa..05d6d56 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_11.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_11.q.out @@ -101,18 +101,15 @@ STAGE PLANS: expressions: key (type: string), 2 (type: int) outputColumnNames: _col0, _col1 Select Operator - expressions: _col0 (type: string), _col1 (type: int) + expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint) outputColumnNames: _col0, _col1 - Select Operator - expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint) - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1 Map 3 Map Operator Tree: TableScan @@ -138,18 +135,15 @@ STAGE PLANS: expressions: key (type: string), 3 (type: int) outputColumnNames: _col0, _col1 Select Operator - expressions: _col0 (type: string), _col1 (type: int) + expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint) outputColumnNames: _col0, _col1 - Select Operator - expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint) - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1 Union 2 Vertex: Union 2 diff --git a/ql/src/test/results/clientpositive/spark/union_remove_19.q.out b/ql/src/test/results/clientpositive/spark/union_remove_19.q.out index ea6ff41..48120ec 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_19.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_19.q.out @@ -342,10 +342,10 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Select Operator - expressions: _col0 (type: string), _col1 (type: bigint) - outputColumnNames: _col0, _col1 + expressions: _col1 (type: bigint) + outputColumnNames: _col1 Select Operator - expressions: _col0 (type: string), _col1 (type: bigint) + expressions: '7' (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 File Output Operator compressed: false @@ -362,10 +362,10 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Select Operator - expressions: _col0 (type: string), _col1 (type: bigint) - outputColumnNames: _col0, _col1 + expressions: _col1 (type: bigint) + outputColumnNames: _col1 Select Operator - expressions: _col0 (type: string), _col1 (type: bigint) + expressions: '7' (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 File Output Operator compressed: false diff --git a/ql/src/test/results/clientpositive/spark/union_remove_25.q.out b/ql/src/test/results/clientpositive/spark/union_remove_25.q.out index 89b8bac..58f764e 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_25.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_25.q.out @@ -311,8 +311,8 @@ STAGE PLANS: alias: srcpart Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string), ds (type: string) - outputColumnNames: _col0, _col1, _col2 + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 500 @@ -320,15 +320,15 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 500 Data size: 5000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + value expressions: _col0 (type: string), _col1 (type: string) Map 4 Map Operator Tree: TableScan alias: srcpart Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string), ds (type: string) - outputColumnNames: _col0, _col1, _col2 + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 500 @@ -336,16 +336,16 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 500 Data size: 5000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + value expressions: _col0 (type: string), _col1 (type: string) Reducer 2 Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string) - outputColumnNames: _col0, _col1, _col2 + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 Limit Number of rows: 500 Select Operator - expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), _col2 (type: string) + expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), '2008-04-08' (type: string) outputColumnNames: _col0, _col1, _col2 File Output Operator compressed: false @@ -357,12 +357,12 @@ STAGE PLANS: Reducer 5 Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string) - outputColumnNames: _col0, _col1, _col2 + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 Limit Number of rows: 500 Select Operator - expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), _col2 (type: string) + expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), '2008-04-08' (type: string) outputColumnNames: _col0, _col1, _col2 File Output Operator compressed: false @@ -499,8 +499,8 @@ STAGE PLANS: alias: srcpart Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + expressions: key (type: string), value (type: string), hr (type: string) + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 1000 @@ -508,15 +508,15 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1000 Data size: 10000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string) Map 4 Map Operator Tree: TableScan alias: srcpart Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + expressions: key (type: string), value (type: string), hr (type: string) + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 1000 @@ -524,16 +524,16 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1000 Data size: 10000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string) Reducer 2 Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col3 (type: string) + outputColumnNames: _col0, _col1, _col3 Limit Number of rows: 1000 Select Operator - expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), _col2 (type: string), _col3 (type: string) + expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), '2008-04-08' (type: string), _col3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 File Output Operator compressed: false @@ -545,12 +545,12 @@ STAGE PLANS: Reducer 5 Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col3 (type: string) + outputColumnNames: _col0, _col1, _col3 Limit Number of rows: 1000 Select Operator - expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), _col2 (type: string), _col3 (type: string) + expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), '2008-04-08' (type: string), _col3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 File Output Operator compressed: false diff --git a/ql/src/test/results/clientpositive/spark/vector_data_types.q.out b/ql/src/test/results/clientpositive/spark/vector_data_types.q.out index 5758c4b..b9032e2 100644 --- a/ql/src/test/results/clientpositive/spark/vector_data_types.q.out +++ b/ql/src/test/results/clientpositive/spark/vector_data_types.q.out @@ -178,6 +178,17 @@ NULL 409 65536 4294967490 46.97 25.92 false fred miller 2013-03-01 09:11:58.7031 29 331 65539 4294967420 73.18 28.96 true ethan brown 2013-03-01 09:11:58.703094 58.85 zync studies 71 298 65540 4294967510 91.63 9.6 false gabriella zipper 2013-03-01 09:11:58.70314 68.14 geology 89 466 65541 4294967468 85.51 19.79 true calvin davidson 2013-03-01 09:11:58.703113 35.38 philosophy +PREHOOK: query: SELECT SUM(HASH(*)) +FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i) as q +PREHOOK: type: QUERY +PREHOOK: Input: default@over1korc +#### A masked pattern was here #### +POSTHOOK: query: SELECT SUM(HASH(*)) +FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i) as q +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1korc +#### A masked pattern was here #### +-17045922556 PREHOOK: query: EXPLAIN select t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i LIMIT 20 PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN select t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i LIMIT 20 @@ -240,23 +251,34 @@ POSTHOOK: query: SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc OR POSTHOOK: type: QUERY POSTHOOK: Input: default@over1korc #### A masked pattern was here #### -NULL 409 65536 4294967490 46.97 25.92 false fred miller 1860-11-12 20:05:55.011315936 33.45 history -18 280 65536 4294967320 32.92 45.94 false holly white 1860-11-12 20:05:55.011285936 58.86 topology -69 489 65536 4294967404 33.52 17.99 false oscar ichabod 1860-11-12 20:05:55.011446936 32.68 topology -108 301 65536 4294967357 90.05 17.59 true ethan johnson 1860-11-12 20:05:55.011470936 75.70 undecided -27 405 65536 4294967508 82.24 29.41 true oscar ovid 1860-11-12 20:05:55.011365936 16.85 biologyd -42 495 65536 4294967431 43.57 46.81 false tom johnson 1860-11-12 20:05:55.011444936 62.25 american history -118 497 65536 4294967381 50.32 12.72 false david nixon 1860-11-12 20:05:55.011484936 83.48 values clariffication -72 267 65537 4294967460 55.2 42.89 false oscar carson 1860-11-12 20:05:55.011352936 51.91 topologyariffication -22 264 65537 4294967419 8.07 10.71 false david xylophone 1860-11-12 20:05:55.011335936 11.87 undecidedriffication -63 458 65537 4294967463 21.94 49.71 true fred van buren 1860-11-12 20:05:55.011477936 99.34 wind surfingfication -121 355 65537 4294967437 85.9 10.99 true calvin brown 1860-11-12 20:05:55.011453936 4.49 yard dutyngfication -59 431 65537 4294967326 11.34 2.9 true oscar robinson 1860-11-12 20:05:55.011360936 81.04 zync studiesfication -83 353 65538 4294967339 0.08 23.91 true holly thompson 1860-11-12 20:05:55.011356936 30.27 quiet hoursfication -109 376 65538 4294967453 59.61 35.62 true priscilla xylophone 1860-11-12 20:05:55.011485936 9.53 study skillsfication -101 283 65538 4294967527 84.03 27.29 true yuri brown 1860-11-12 20:05:55.011505936 44.24 biologyillsfication -120 331 65539 4294967324 88.02 40.94 true holly nixon 1860-11-12 20:05:55.011461936 96.64 yard dutylsfication -70 430 65539 4294967534 18.89 43.84 true tom carson 1860-11-12 20:05:55.011381936 21.93 joggyinglsfication -29 331 65539 4294967420 73.18 28.96 true ethan brown 1860-11-12 20:05:55.011293936 58.85 zync studiesfication -71 298 65540 4294967510 91.63 9.6 false gabriella zipper 1860-11-12 20:05:55.011339936 68.14 geologydiesfication -89 466 65541 4294967468 85.51 19.79 true calvin davidson 1860-11-12 20:05:55.011312936 35.38 philosophysfication +NULL 409 65536 4294967490 46.97 25.92 false fred miller 2013-03-01 09:11:58.703116 33.45 history +18 280 65536 4294967320 32.92 45.94 false holly white 2013-03-01 09:11:58.703086 58.86 topology +69 489 65536 4294967404 33.52 17.99 false oscar ichabod 2013-03-01 09:11:58.703247 32.68 topology +108 301 65536 4294967357 90.05 17.59 true ethan johnson 2013-03-01 09:11:58.703271 75.70 undecided +27 405 65536 4294967508 82.24 29.41 true oscar ovid 2013-03-01 09:11:58.703166 16.85 biology +42 495 65536 4294967431 43.57 46.81 false tom johnson 2013-03-01 09:11:58.703245 62.25 american history +118 497 65536 4294967381 50.32 12.72 false david nixon 2013-03-01 09:11:58.703285 83.48 values clariffication +72 267 65537 4294967460 55.2 42.89 false oscar carson 2013-03-01 09:11:58.703153 51.91 topology +22 264 65537 4294967419 8.07 10.71 false david xylophone 2013-03-01 09:11:58.703136 11.87 undecided +63 458 65537 4294967463 21.94 49.71 true fred van buren 2013-03-01 09:11:58.703278 99.34 wind surfing +121 355 65537 4294967437 85.9 10.99 true calvin brown 2013-03-01 09:11:58.703254 4.49 yard duty +59 431 65537 4294967326 11.34 2.9 true oscar robinson 2013-03-01 09:11:58.703161 81.04 zync studies +83 353 65538 4294967339 0.08 23.91 true holly thompson 2013-03-01 09:11:58.703157 30.27 quiet hour +109 376 65538 4294967453 59.61 35.62 true priscilla xylophone 2013-03-01 09:11:58.703286 9.53 study skills +101 283 65538 4294967527 84.03 27.29 true yuri brown 2013-03-01 09:11:58.703306 44.24 biology +120 331 65539 4294967324 88.02 40.94 true holly nixon 2013-03-01 09:11:58.703262 96.64 yard duty +70 430 65539 4294967534 18.89 43.84 true tom carson 2013-03-01 09:11:58.703182 21.93 joggying +29 331 65539 4294967420 73.18 28.96 true ethan brown 2013-03-01 09:11:58.703094 58.85 zync studies +71 298 65540 4294967510 91.63 9.6 false gabriella zipper 2013-03-01 09:11:58.70314 68.14 geology +89 466 65541 4294967468 85.51 19.79 true calvin davidson 2013-03-01 09:11:58.703113 35.38 philosophy +PREHOOK: query: SELECT SUM(HASH(*)) +FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i) as q +PREHOOK: type: QUERY +PREHOOK: Input: default@over1korc +#### A masked pattern was here #### +POSTHOOK: query: SELECT SUM(HASH(*)) +FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i) as q +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1korc +#### A masked pattern was here #### +-17045922556 diff --git a/ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out b/ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out index 3c6d561..55a2738 100644 --- a/ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out +++ b/ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out @@ -114,14 +114,14 @@ POSTHOOK: query: SELECT cint, POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_vgby #### A masked pattern was here #### -253665376 1024 9767.0054054054 -9779.5486486487 -347484.0818378374 1024 11697.96923076923100 -11712.99230769231000 -416182.64030769233089 762 2 5831542.2692483780 1531.2194054054 5833073.4886537834 2 6984454.21109769200000 1833.9456923076925 6986288.15678999969250 -528534767 1024 5831542.2692483780 -9777.1594594595 11646372.8607481068 1024 6984454.21109769200000 -11710.13076923077100 13948892.79980307629003 +253665376 1024 9767.0054054054 -9779.5486486487 -347484.0818378374 1024 11697.96923076923100 -11712.99230769231000 -416182.64030769233089 +-3728 6 5831542.2692483780 -3367.6517567568 5817556.0411483778 6 6984454.21109769200000 -4033.445769230769 6967702.86724384584710 +-563 2 -515.6210729730 -3367.6517567568 -3883.2728297298 2 -617.56077692307690 -4033.445769230769 -4651.00654615384590 6981 3 5831542.269248378 -515.6210729730 5830511.0271024320 3 6984454.211097692 -617.56077692307690 6983219.08954384584620 626923679 1024 9723.4027027027 -9778.9513513514 10541.0525297287 1024 11645.74615384615400 -11712.27692307692300 12625.04759999997746 +528534767 1024 5831542.2692483780 -9777.1594594595 11646372.8607481068 1024 6984454.21109769200000 -11710.13076923077100 13948892.79980307629003 NULL 3072 9318.4351351351 -4298.1513513514 5018444.1081079808 3072 11160.71538461538500 -5147.90769230769300 6010604.30769230735360 --3728 6 5831542.2692483780 -3367.6517567568 5817556.0411483778 6 6984454.21109769200000 -4033.445769230769 6967702.86724384584710 --563 2 -515.6210729730 -3367.6517567568 -3883.2728297298 2 -617.56077692307690 -4033.445769230769 -4651.00654615384590 PREHOOK: query: -- Now add the others... EXPLAIN SELECT cint, COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), AVG(cdecimal1), STDDEV_POP(cdecimal1), STDDEV_SAMP(cdecimal1), @@ -218,11 +218,11 @@ POSTHOOK: query: SELECT cint, POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_vgby #### A masked pattern was here #### -253665376 1024 9767.0054054054 -9779.5486486487 -347484.0818378374 -339.33992366976309 5708.9563478862 5711.745967572779 1024 11697.96923076923100 -11712.99230769231000 -416182.64030769233089 -406.428359675480791885 6837.632716002934 6840.973851172274 762 2 5831542.2692483780 1531.2194054054 5833073.4886537834 2916536.7443268917 2915005.5249214866 4122440.3477364695 2 6984454.21109769200000 1833.9456923076925 6986288.15678999969250 3493144.07839499984625 3491310.1327026924 4937458.140118758 -528534767 1024 5831542.2692483780 -9777.1594594595 11646372.8607481068 11373.41099682432305 257528.92988206653 257654.7686043977 1024 6984454.21109769200000 -11710.13076923077100 13948892.79980307629003 13621.965624807691689482 308443.1074570801 308593.82484083984 +253665376 1024 9767.0054054054 -9779.5486486487 -347484.0818378374 -339.33992366976309 5708.9563478862 5711.745967572779 1024 11697.96923076923100 -11712.99230769231000 -416182.64030769233089 -406.428359675480791885 6837.632716002934 6840.973851172274 +-3728 6 5831542.2692483780 -3367.6517567568 5817556.0411483778 969592.67352472963333 2174330.2092403853 2381859.406131774 6 6984454.21109769200000 -4033.445769230769 6967702.86724384584710 1161283.811207307641183333 2604201.2704476737 2852759.5602156054 +-563 2 -515.6210729730 -3367.6517567568 -3883.2728297298 -1941.6364148649 1426.0153418918999 2016.6902366556308 2 -617.56077692307690 -4033.445769230769 -4651.00654615384590 -2325.50327307692295 1707.9424961538462 2415.395441814127 6981 3 5831542.269248378 -515.6210729730 5830511.0271024320 1943503.67570081066667 2749258.455012492 3367140.1929065133 3 6984454.211097692 -617.56077692307690 6983219.08954384584620 2327739.696514615282066667 3292794.4113115156 4032833.0678006653 626923679 1024 9723.4027027027 -9778.9513513514 10541.0525297287 10.29399661106318 5742.09145323734 5744.897264034267 1024 11645.74615384615400 -11712.27692307692300 12625.04759999997746 12.329148046874977988 6877.318722794877 6880.679250101603 +528534767 1024 5831542.2692483780 -9777.1594594595 11646372.8607481068 11373.41099682432305 257528.92988206653 257654.7686043977 1024 6984454.21109769200000 -11710.13076923077100 13948892.79980307629003 13621.965624807691689482 308443.1074570801 308593.82484083984 NULL 3072 9318.4351351351 -4298.1513513514 5018444.1081079808 1633.60810810806667 5695.483082135364 5696.4103077145055 3072 11160.71538461538500 -5147.90769230769300 6010604.30769230735360 1956.576923076922966667 6821.495748565159 6822.606289190924 --3728 6 5831542.2692483780 -3367.6517567568 5817556.0411483778 969592.67352472963333 2174330.2092403853 2381859.406131774 6 6984454.21109769200000 -4033.445769230769 6967702.86724384584710 1161283.811207307641183333 2604201.2704476737 2852759.5602156054 --563 2 -515.6210729730 -3367.6517567568 -3883.2728297298 -1941.6364148649 1426.0153418918999 2016.6902366556308 2 -617.56077692307690 -4033.445769230769 -4651.00654615384590 -2325.50327307692295 1707.9424961538462 2415.395441814127 diff --git a/ql/src/test/results/clientpositive/spark/vector_decimal_mapjoin.q.out b/ql/src/test/results/clientpositive/spark/vector_decimal_mapjoin.q.out index a81f0a7..5a10390 100644 --- a/ql/src/test/results/clientpositive/spark/vector_decimal_mapjoin.q.out +++ b/ql/src/test/results/clientpositive/spark/vector_decimal_mapjoin.q.out @@ -48,12 +48,12 @@ STAGE PLANS: alias: r Statistics: Num rows: 1100 Data size: 127658 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (cint is not null and (cint = 6981)) (type: boolean) - Statistics: Num rows: 275 Data size: 31914 Basic stats: COMPLETE Column stats: NONE + predicate: (cint = 6981) (type: boolean) + Statistics: Num rows: 550 Data size: 63829 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: 6981 (type: int) sort order: + - Statistics: Num rows: 275 Data size: 31914 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 550 Data size: 63829 Basic stats: COMPLETE Column stats: NONE value expressions: cdecimal2 (type: decimal(23,14)) Execution mode: vectorized Map 3 @@ -62,12 +62,12 @@ STAGE PLANS: alias: l Statistics: Num rows: 1100 Data size: 127658 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (cint is not null and (cint = 6981)) (type: boolean) - Statistics: Num rows: 275 Data size: 31914 Basic stats: COMPLETE Column stats: NONE + predicate: (cint = 6981) (type: boolean) + Statistics: Num rows: 550 Data size: 63829 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: 6981 (type: int) sort order: + - Statistics: Num rows: 275 Data size: 31914 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 550 Data size: 63829 Basic stats: COMPLETE Column stats: NONE value expressions: cdecimal1 (type: decimal(20,10)) Execution mode: vectorized Reducer 2 @@ -79,14 +79,14 @@ STAGE PLANS: 0 {VALUE._col1} 1 {VALUE._col2} outputColumnNames: _col1, _col9 - Statistics: Num rows: 302 Data size: 35105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 605 Data size: 70211 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 6981 (type: int), 6981 (type: int), _col1 (type: decimal(20,10)), _col9 (type: decimal(23,14)) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 302 Data size: 35105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 605 Data size: 70211 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 302 Data size: 35105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 605 Data size: 70211 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/spark/vectorization_13.q.out b/ql/src/test/results/clientpositive/spark/vectorization_13.q.out index 8e0c7bc..923d805 100644 --- a/ql/src/test/results/clientpositive/spark/vectorization_13.q.out +++ b/ql/src/test/results/clientpositive/spark/vectorization_13.q.out @@ -193,46 +193,46 @@ LIMIT 40 POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypesorc #### A masked pattern was here #### -NULL -25 1969-12-31 16:00:13.374 -25.0 NULL 25 -25 0 -25.0 -0.0 25.0 -1988.8249999999998 0.0 25.0 0.0 -10.175 -25.0 1.0512000000000001 -25.0 0.0 -25 -NULL 62 1969-12-31 16:00:13.524 62.0 NULL -62 62 0 62.0 0.0 -62.0 4932.286 0.0 -62.0 0.0 -10.175 62.0 -0.4238709677419355 62.0 0.0 62 -NULL -38 1969-12-31 16:00:14.548 -38.0 NULL 38 -38 0 -38.0 -0.0 38.0 -3023.014 0.0 38.0 0.0 -10.175 -38.0 0.6915789473684211 -38.0 0.0 -38 -true 26 1969-12-31 16:00:14.341 26.0 1cGVWH7n1QU -26 26 0 26.0 0.0 -26.0 2068.3779999999997 0.0 -26.0 0.0 -10.175 26.0 -1.0107692307692309 26.0 0.0 26 -NULL -62 1969-12-31 16:00:12.388 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62 -NULL 28 1969-12-31 16:00:12.148 28.0 NULL -28 28 0 28.0 0.0 -28.0 2227.484 0.0 -28.0 0.0 -10.175 28.0 -0.9385714285714286 28.0 0.0 28 -true 49 1969-12-31 16:00:12.273 49.0 821UdmGbkEf4j -49 49 0 49.0 0.0 -49.0 3898.0969999999998 0.0 -49.0 0.0 -10.175 49.0 -0.5363265306122449 49.0 0.0 49 -true 13 1969-12-31 16:00:12.554 13.0 cvLH6Eat2yFsyy7p -13 13 0 13.0 0.0 -13.0 1034.1889999999999 0.0 -13.0 0.0 -10.175 13.0 -2.0215384615384617 13.0 0.0 13 -NULL 41 1969-12-31 16:00:14.017 41.0 NULL -41 41 0 41.0 0.0 -41.0 3261.673 0.0 -41.0 0.0 -10.175 41.0 -0.6409756097560976 41.0 0.0 41 -NULL -62 1969-12-31 16:00:14.247 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62 -NULL -49 1969-12-31 16:00:11.627 -49.0 NULL 49 -49 0 -49.0 -0.0 49.0 -3898.0969999999998 0.0 49.0 0.0 -10.175 -49.0 0.5363265306122449 -49.0 0.0 -49 -NULL 40 1969-12-31 16:00:14.977 40.0 NULL -40 40 0 40.0 0.0 -40.0 3182.12 0.0 -40.0 0.0 -10.175 40.0 -0.657 40.0 0.0 40 -NULL 13 1969-12-31 16:00:14.548 13.0 NULL -13 13 0 13.0 0.0 -13.0 1034.1889999999999 0.0 -13.0 0.0 -10.175 13.0 -2.0215384615384617 13.0 0.0 13 -NULL -2 1969-12-31 16:00:14.391 -2.0 NULL 2 -2 0 -2.0 -0.0 2.0 -159.106 0.0 2.0 0.0 -10.175 -2.0 13.14 -2.0 0.0 -2 -true -22 1969-12-31 16:00:14.318 -22.0 1cGVWH7n1QU 22 -22 0 -22.0 -0.0 22.0 -1750.166 0.0 22.0 0.0 -10.175 -22.0 1.1945454545454546 -22.0 0.0 -22 -true 34 1969-12-31 16:00:13.221 34.0 821UdmGbkEf4j -34 34 0 34.0 0.0 -34.0 2704.8019999999997 0.0 -34.0 0.0 -10.175 34.0 -0.7729411764705882 34.0 0.0 34 -NULL 18 1969-12-31 16:00:12.437 18.0 NULL -18 18 0 18.0 0.0 -18.0 1431.954 0.0 -18.0 0.0 -10.175 18.0 -1.46 18.0 0.0 18 -NULL 62 1969-12-31 16:00:14.982 62.0 NULL -62 62 0 62.0 0.0 -62.0 4932.286 0.0 -62.0 0.0 -10.175 62.0 -0.4238709677419355 62.0 0.0 62 -true 38 1969-12-31 16:00:14.239 38.0 821UdmGbkEf4j -38 38 0 38.0 0.0 -38.0 3023.014 0.0 -38.0 0.0 -10.175 38.0 -0.6915789473684211 38.0 0.0 38 -true 61 1969-12-31 16:00:15.673 61.0 1cGVWH7n1QU -61 61 0 61.0 0.0 -61.0 4852.733 0.0 -61.0 0.0 -10.175 61.0 -0.4308196721311476 61.0 0.0 61 -true 9 1969-12-31 16:00:11.544 9.0 1cGVWH7n1QU -9 9 0 9.0 0.0 -9.0 715.977 0.0 -9.0 0.0 -10.175 9.0 -2.92 9.0 0.0 9 -NULL -51 1969-12-31 16:00:12.494 -51.0 NULL 51 -51 0 -51.0 -0.0 51.0 -4057.203 0.0 51.0 0.0 -10.175 -51.0 0.5152941176470588 -51.0 0.0 -51 -true 25 1969-12-31 16:00:13.338 25.0 cvLH6Eat2yFsyy7p -25 25 0 25.0 0.0 -25.0 1988.8249999999998 0.0 -25.0 0.0 -10.175 25.0 -1.0512000000000001 25.0 0.0 25 -true -14 1969-12-31 16:00:15.625 -14.0 cvLH6Eat2yFsyy7p 14 -14 0 -14.0 -0.0 14.0 -1113.742 0.0 14.0 0.0 -10.175 -14.0 1.8771428571428572 -14.0 0.0 -14 -true -34 1969-12-31 16:00:15.254 -34.0 821UdmGbkEf4j 34 -34 0 -34.0 -0.0 34.0 -2704.8019999999997 0.0 34.0 0.0 -10.175 -34.0 0.7729411764705882 -34.0 0.0 -34 -true -1 1969-12-31 16:00:16.189 -1.0 cvLH6Eat2yFsyy7p 1 -1 0 -1.0 -0.0 1.0 -79.553 0.0 1.0 0.0 -10.175 -1.0 26.28 -1.0 0.0 -1 -NULL -30 1969-12-31 16:00:12.935 -30.0 NULL 30 -30 0 -30.0 -0.0 30.0 -2386.59 0.0 30.0 0.0 -10.175 -30.0 0.876 -30.0 0.0 -30 -true -53 1969-12-31 16:00:11.977 -53.0 821UdmGbkEf4j 53 -53 0 -53.0 -0.0 53.0 -4216.309 0.0 53.0 0.0 -10.175 -53.0 0.4958490566037736 -53.0 0.0 -53 -true 6 1969-12-31 16:00:11.357 6.0 cvLH6Eat2yFsyy7p -6 6 0 6.0 0.0 -6.0 477.318 0.0 -6.0 0.0 -10.175 6.0 -4.38 6.0 0.0 6 -NULL 30 1969-12-31 16:00:11.876 30.0 NULL -30 30 0 30.0 0.0 -30.0 2386.59 0.0 -30.0 0.0 -10.175 30.0 -0.876 30.0 0.0 30 -NULL 48 1969-12-31 16:00:15.512 48.0 NULL -48 48 0 48.0 0.0 -48.0 3818.544 0.0 -48.0 0.0 -10.175 48.0 -0.5475 48.0 0.0 48 -NULL -21 1969-12-31 16:00:13.32 -21.0 NULL 21 -21 0 -21.0 -0.0 21.0 -1670.6129999999998 0.0 21.0 0.0 -10.175 -21.0 1.2514285714285716 -21.0 0.0 -21 -NULL -42 1969-12-31 16:00:16.191 -42.0 NULL 42 -42 0 -42.0 -0.0 42.0 -3341.2259999999997 0.0 42.0 0.0 -10.175 -42.0 0.6257142857142858 -42.0 0.0 -42 -NULL 0 1969-12-31 16:00:11.089 0.0 NULL 0 0 0 0.0 0.0 -0.0 0.0 0.0 -0.0 0.0 -10.175 0.0 NULL 0.0 NULL 0 -true -58 1969-12-31 16:00:13.209 -58.0 cvLH6Eat2yFsyy7p 58 -58 0 -58.0 -0.0 58.0 -4614.074 0.0 58.0 0.0 -10.175 -58.0 0.4531034482758621 -58.0 0.0 -58 -NULL 60 1969-12-31 16:00:11.003 60.0 NULL -60 60 0 60.0 0.0 -60.0 4773.18 0.0 -60.0 0.0 -10.175 60.0 -0.438 60.0 0.0 60 -true 23 1969-12-31 16:00:11.343 23.0 cvLH6Eat2yFsyy7p -23 23 0 23.0 0.0 -23.0 1829.719 0.0 -23.0 0.0 -10.175 23.0 -1.142608695652174 23.0 0.0 23 -NULL -27 1969-12-31 16:00:14.065 -27.0 NULL 27 -27 0 -27.0 -0.0 27.0 -2147.931 0.0 27.0 0.0 -10.175 -27.0 0.9733333333333334 -27.0 0.0 -27 -true 57 1969-12-31 16:00:13.71 57.0 cvLH6Eat2yFsyy7p -57 57 0 57.0 0.0 -57.0 4534.521 0.0 -57.0 0.0 -10.175 57.0 -0.4610526315789474 57.0 0.0 57 -true -59 1969-12-31 16:00:12.227 -59.0 cvLH6Eat2yFsyy7p 59 -59 0 -59.0 -0.0 59.0 -4693.6269999999995 0.0 59.0 0.0 -10.175 -59.0 0.44542372881355935 -59.0 0.0 -59 +NULL -31 1969-12-31 16:00:16.059 -31.0 NULL 31 -31 0 -31.0 -0.0 31.0 -2466.143 0.0 31.0 0.0 -10.175 -31.0 0.847741935483871 -31.0 0.0 -31 +true 50 1969-12-31 16:00:16.338 50.0 821UdmGbkEf4j -50 50 0 50.0 0.0 -50.0 3977.6499999999996 0.0 -50.0 0.0 -10.175 50.0 -0.5256000000000001 50.0 0.0 50 +NULL 23 1969-12-31 16:00:11.785 23.0 NULL -23 23 0 23.0 0.0 -23.0 1829.719 0.0 -23.0 0.0 -10.175 23.0 -1.142608695652174 23.0 0.0 23 +true 14 1969-12-31 16:00:11.528 14.0 821UdmGbkEf4j -14 14 0 14.0 0.0 -14.0 1113.742 0.0 -14.0 0.0 -10.175 14.0 -1.8771428571428572 14.0 0.0 14 +true 50 1969-12-31 16:00:11.087 50.0 1cGVWH7n1QU -50 50 0 50.0 0.0 -50.0 3977.6499999999996 0.0 -50.0 0.0 -10.175 50.0 -0.5256000000000001 50.0 0.0 50 +NULL 13 1969-12-31 16:00:12.498 13.0 NULL -13 13 0 13.0 0.0 -13.0 1034.1889999999999 0.0 -13.0 0.0 -10.175 13.0 -2.0215384615384617 13.0 0.0 13 +true 36 1969-12-31 16:00:15.145 36.0 1cGVWH7n1QU -36 36 0 36.0 0.0 -36.0 2863.908 0.0 -36.0 0.0 -10.175 36.0 -0.73 36.0 0.0 36 +true 3 1969-12-31 16:00:14.349 3.0 cvLH6Eat2yFsyy7p -3 3 0 3.0 0.0 -3.0 238.659 0.0 -3.0 0.0 -10.175 3.0 -8.76 3.0 0.0 3 +NULL -4 1969-12-31 16:00:13.237 -4.0 NULL 4 -4 0 -4.0 -0.0 4.0 -318.212 0.0 4.0 0.0 -10.175 -4.0 6.57 -4.0 0.0 -4 +NULL -20 1969-12-31 16:00:13.131 -20.0 NULL 20 -20 0 -20.0 -0.0 20.0 -1591.06 0.0 20.0 0.0 -10.175 -20.0 1.314 -20.0 0.0 -20 +true 30 1969-12-31 16:00:15.705 30.0 821UdmGbkEf4j -30 30 0 30.0 0.0 -30.0 2386.59 0.0 -30.0 0.0 -10.175 30.0 -0.876 30.0 0.0 30 +true -57 1969-12-31 16:00:13.365 -57.0 1cGVWH7n1QU 57 -57 0 -57.0 -0.0 57.0 -4534.521 0.0 57.0 0.0 -10.175 -57.0 0.4610526315789474 -57.0 0.0 -57 +true 40 1969-12-31 16:00:16.188 40.0 1cGVWH7n1QU -40 40 0 40.0 0.0 -40.0 3182.12 0.0 -40.0 0.0 -10.175 40.0 -0.657 40.0 0.0 40 +NULL -21 1969-12-31 16:00:12.544 -21.0 NULL 21 -21 0 -21.0 -0.0 21.0 -1670.6129999999998 0.0 21.0 0.0 -10.175 -21.0 1.2514285714285716 -21.0 0.0 -21 +NULL 7 1969-12-31 16:00:14.872 7.0 NULL -7 7 0 7.0 0.0 -7.0 556.871 0.0 -7.0 0.0 -10.175 7.0 -3.7542857142857144 7.0 0.0 7 +true -34 1969-12-31 16:00:14.043 -34.0 821UdmGbkEf4j 34 -34 0 -34.0 -0.0 34.0 -2704.8019999999997 0.0 34.0 0.0 -10.175 -34.0 0.7729411764705882 -34.0 0.0 -34 +NULL -58 1969-12-31 16:00:15.658 -58.0 NULL 58 -58 0 -58.0 -0.0 58.0 -4614.074 0.0 58.0 0.0 -10.175 -58.0 0.4531034482758621 -58.0 0.0 -58 +true -59 1969-12-31 16:00:11.231 -59.0 821UdmGbkEf4j 59 -59 0 -59.0 -0.0 59.0 -4693.6269999999995 0.0 59.0 0.0 -10.175 -59.0 0.44542372881355935 -59.0 0.0 -59 +NULL 44 1969-12-31 16:00:16.279 44.0 NULL -44 44 0 44.0 0.0 -44.0 3500.332 0.0 -44.0 0.0 -10.175 44.0 -0.5972727272727273 44.0 0.0 44 +true -27 1969-12-31 16:00:14.723 -27.0 1cGVWH7n1QU 27 -27 0 -27.0 -0.0 27.0 -2147.931 0.0 27.0 0.0 -10.175 -27.0 0.9733333333333334 -27.0 0.0 -27 +NULL 0 1969-12-31 16:00:13.396 0.0 NULL 0 0 0 0.0 0.0 -0.0 0.0 0.0 -0.0 0.0 -10.175 0.0 NULL 0.0 NULL 0 +true -25 1969-12-31 16:00:15.516 -25.0 821UdmGbkEf4j 25 -25 0 -25.0 -0.0 25.0 -1988.8249999999998 0.0 25.0 0.0 -10.175 -25.0 1.0512000000000001 -25.0 0.0 -25 +true -40 1969-12-31 16:00:11.833 -40.0 cvLH6Eat2yFsyy7p 40 -40 0 -40.0 -0.0 40.0 -3182.12 0.0 40.0 0.0 -10.175 -40.0 0.657 -40.0 0.0 -40 +NULL 29 1969-12-31 16:00:16.19 29.0 NULL -29 29 0 29.0 0.0 -29.0 2307.037 0.0 -29.0 0.0 -10.175 29.0 -0.9062068965517242 29.0 0.0 29 +NULL -27 1969-12-31 16:00:15.435 -27.0 NULL 27 -27 0 -27.0 -0.0 27.0 -2147.931 0.0 27.0 0.0 -10.175 -27.0 0.9733333333333334 -27.0 0.0 -27 +NULL -63 1969-12-31 16:00:15.436 -63.0 NULL 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63 +NULL -6 1969-12-31 16:00:11.883 -6.0 NULL 6 -6 0 -6.0 -0.0 6.0 -477.318 0.0 6.0 0.0 -10.175 -6.0 4.38 -6.0 0.0 -6 +true 14 1969-12-31 16:00:11.302 14.0 821UdmGbkEf4j -14 14 0 14.0 0.0 -14.0 1113.742 0.0 -14.0 0.0 -10.175 14.0 -1.8771428571428572 14.0 0.0 14 +true -63 1969-12-31 16:00:14.899 -63.0 cvLH6Eat2yFsyy7p 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63 +true -13 1969-12-31 16:00:15.223 -13.0 1cGVWH7n1QU 13 -13 0 -13.0 -0.0 13.0 -1034.1889999999999 0.0 13.0 0.0 -10.175 -13.0 2.0215384615384617 -13.0 0.0 -13 +true 49 1969-12-31 16:00:15.541 49.0 cvLH6Eat2yFsyy7p -49 49 0 49.0 0.0 -49.0 3898.0969999999998 0.0 -49.0 0.0 -10.175 49.0 -0.5363265306122449 49.0 0.0 49 +true -60 1969-12-31 16:00:12.291 -60.0 821UdmGbkEf4j 60 -60 0 -60.0 -0.0 60.0 -4773.18 0.0 60.0 0.0 -10.175 -60.0 0.438 -60.0 0.0 -60 +NULL 32 1969-12-31 16:00:14.026 32.0 NULL -32 32 0 32.0 0.0 -32.0 2545.696 0.0 -32.0 0.0 -10.175 32.0 -0.82125 32.0 0.0 32 +true 16 1969-12-31 16:00:15.698 16.0 1cGVWH7n1QU -16 16 0 16.0 0.0 -16.0 1272.848 0.0 -16.0 0.0 -10.175 16.0 -1.6425 16.0 0.0 16 +NULL 57 1969-12-31 16:00:14.011 57.0 NULL -57 57 0 57.0 0.0 -57.0 4534.521 0.0 -57.0 0.0 -10.175 57.0 -0.4610526315789474 57.0 0.0 57 +true 41 1969-12-31 16:00:12.235 41.0 cvLH6Eat2yFsyy7p -41 41 0 41.0 0.0 -41.0 3261.673 0.0 -41.0 0.0 -10.175 41.0 -0.6409756097560976 41.0 0.0 41 +true -64 1969-12-31 16:00:12.857 -64.0 821UdmGbkEf4j 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64 +true 36 1969-12-31 16:00:13.825 36.0 cvLH6Eat2yFsyy7p -36 36 0 36.0 0.0 -36.0 2863.908 0.0 -36.0 0.0 -10.175 36.0 -0.73 36.0 0.0 36 +NULL -15 1969-12-31 16:00:15.186 -15.0 NULL 15 -15 0 -15.0 -0.0 15.0 -1193.295 0.0 15.0 0.0 -10.175 -15.0 1.752 -15.0 0.0 -15 +true -2 1969-12-31 16:00:15.431 -2.0 1cGVWH7n1QU 2 -2 0 -2.0 -0.0 2.0 -159.106 0.0 2.0 0.0 -10.175 -2.0 13.14 -2.0 0.0 -2 PREHOOK: query: -- double compare timestamp EXPLAIN SELECT cboolean1, @@ -430,43 +430,43 @@ LIMIT 40 POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypesorc #### A masked pattern was here #### -true -21 1969-12-31 16:00:00.826 -21.0 1cGVWH7n1QU 21 -21 0 -21.0 -0.0 21.0 -1670.6129999999998 0.0 21.0 0.0 -10.175 -21.0 1.2514285714285716 -21.0 0.0 -21 -true 26 1969-12-31 16:00:14.341 26.0 1cGVWH7n1QU -26 26 0 26.0 0.0 -26.0 2068.3779999999997 0.0 -26.0 0.0 -10.175 26.0 -1.0107692307692309 26.0 0.0 26 -NULL -62 1969-12-31 16:00:12.388 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62 -NULL -16 1969-12-31 15:59:58.727 -16.0 NULL 16 -16 0 -16.0 -0.0 16.0 -1272.848 0.0 16.0 0.0 -10.175 -16.0 1.6425 -16.0 0.0 -16 -NULL -24 1969-12-31 16:00:07.302 -24.0 NULL 24 -24 0 -24.0 -0.0 24.0 -1909.272 0.0 24.0 0.0 -10.175 -24.0 1.095 -24.0 0.0 -24 -true -10 1969-12-31 16:00:05.636 -10.0 1cGVWH7n1QU 10 -10 0 -10.0 -0.0 10.0 -795.53 0.0 10.0 0.0 -10.175 -10.0 2.628 -10.0 0.0 -10 -NULL 41 1969-12-31 16:00:14.017 41.0 NULL -41 41 0 41.0 0.0 -41.0 3261.673 0.0 -41.0 0.0 -10.175 41.0 -0.6409756097560976 41.0 0.0 41 -NULL 26 1969-12-31 16:00:04.699 26.0 NULL -26 26 0 26.0 0.0 -26.0 2068.3779999999997 0.0 -26.0 0.0 -10.175 26.0 -1.0107692307692309 26.0 0.0 26 -NULL 18 1969-12-31 16:00:02.295 18.0 NULL -18 18 0 18.0 0.0 -18.0 1431.954 0.0 -18.0 0.0 -10.175 18.0 -1.46 18.0 0.0 18 -true 61 1969-12-31 16:00:05.394 61.0 cvLH6Eat2yFsyy7p -61 61 0 61.0 0.0 -61.0 4852.733 0.0 -61.0 0.0 -10.175 61.0 -0.4308196721311476 61.0 0.0 61 -true 9 1969-12-31 16:00:11.544 9.0 1cGVWH7n1QU -9 9 0 9.0 0.0 -9.0 715.977 0.0 -9.0 0.0 -10.175 9.0 -2.92 9.0 0.0 9 -true 55 1969-12-31 15:59:58.962 55.0 cvLH6Eat2yFsyy7p -55 55 0 55.0 0.0 -55.0 4375.415 0.0 -55.0 0.0 -10.175 55.0 -0.47781818181818186 55.0 0.0 55 -true 52 1969-12-31 16:00:07.428 52.0 1cGVWH7n1QU -52 52 0 52.0 0.0 -52.0 4136.755999999999 0.0 -52.0 0.0 -10.175 52.0 -0.5053846153846154 52.0 0.0 52 -NULL -41 1969-12-31 16:00:00.096 -41.0 NULL 41 -41 0 -41.0 -0.0 41.0 -3261.673 0.0 41.0 0.0 -10.175 -41.0 0.6409756097560976 -41.0 0.0 -41 -NULL -51 1969-12-31 16:00:12.494 -51.0 NULL 51 -51 0 -51.0 -0.0 51.0 -4057.203 0.0 51.0 0.0 -10.175 -51.0 0.5152941176470588 -51.0 0.0 -51 -NULL -18 1969-12-31 16:00:06.015 -18.0 NULL 18 -18 0 -18.0 -0.0 18.0 -1431.954 0.0 18.0 0.0 -10.175 -18.0 1.46 -18.0 0.0 -18 -true -34 1969-12-31 16:00:15.254 -34.0 821UdmGbkEf4j 34 -34 0 -34.0 -0.0 34.0 -2704.8019999999997 0.0 34.0 0.0 -10.175 -34.0 0.7729411764705882 -34.0 0.0 -34 -NULL 46 1969-12-31 16:00:00.259 46.0 NULL -46 46 0 46.0 0.0 -46.0 3659.438 0.0 -46.0 0.0 -10.175 46.0 -0.571304347826087 46.0 0.0 46 -NULL -30 1969-12-31 16:00:12.935 -30.0 NULL 30 -30 0 -30.0 -0.0 30.0 -2386.59 0.0 30.0 0.0 -10.175 -30.0 0.876 -30.0 0.0 -30 -true 34 1969-12-31 16:00:10.818 34.0 1cGVWH7n1QU -34 34 0 34.0 0.0 -34.0 2704.8019999999997 0.0 -34.0 0.0 -10.175 34.0 -0.7729411764705882 34.0 0.0 34 -true -29 1969-12-31 16:00:03.586 -29.0 1cGVWH7n1QU 29 -29 0 -29.0 -0.0 29.0 -2307.037 0.0 29.0 0.0 -10.175 -29.0 0.9062068965517242 -29.0 0.0 -29 -NULL 48 1969-12-31 16:00:15.512 48.0 NULL -48 48 0 48.0 0.0 -48.0 3818.544 0.0 -48.0 0.0 -10.175 48.0 -0.5475 48.0 0.0 48 -NULL -21 1969-12-31 16:00:13.32 -21.0 NULL 21 -21 0 -21.0 -0.0 21.0 -1670.6129999999998 0.0 21.0 0.0 -10.175 -21.0 1.2514285714285716 -21.0 0.0 -21 -true 15 1969-12-31 16:00:03.901 15.0 1cGVWH7n1QU -15 15 0 15.0 0.0 -15.0 1193.295 0.0 -15.0 0.0 -10.175 15.0 -1.752 15.0 0.0 15 -NULL 53 1969-12-31 16:00:00.557 53.0 NULL -53 53 0 53.0 0.0 -53.0 4216.309 0.0 -53.0 0.0 -10.175 53.0 -0.4958490566037736 53.0 0.0 53 -true -54 1969-12-31 16:00:01.64 -54.0 821UdmGbkEf4j 54 -54 0 -54.0 -0.0 54.0 -4295.862 0.0 54.0 0.0 -10.175 -54.0 0.4866666666666667 -54.0 0.0 -54 -true -44 1969-12-31 16:00:05.272 -44.0 1cGVWH7n1QU 44 -44 0 -44.0 -0.0 44.0 -3500.332 0.0 44.0 0.0 -10.175 -44.0 0.5972727272727273 -44.0 0.0 -44 -true 28 1969-12-31 16:00:02.779 28.0 1cGVWH7n1QU -28 28 0 28.0 0.0 -28.0 2227.484 0.0 -28.0 0.0 -10.175 28.0 -0.9385714285714286 28.0 0.0 28 -NULL 47 1969-12-31 16:00:06.723 47.0 NULL -47 47 0 47.0 0.0 -47.0 3738.991 0.0 -47.0 0.0 -10.175 47.0 -0.5591489361702128 47.0 0.0 47 -NULL 36 1969-12-31 16:00:00.865 36.0 NULL -36 36 0 36.0 0.0 -36.0 2863.908 0.0 -36.0 0.0 -10.175 36.0 -0.73 36.0 0.0 36 -NULL 35 1969-12-31 16:00:09.892 35.0 NULL -35 35 0 35.0 0.0 -35.0 2784.355 0.0 -35.0 0.0 -10.175 35.0 -0.7508571428571429 35.0 0.0 35 -NULL 55 1969-12-31 16:00:15.035 55.0 NULL -55 55 0 55.0 0.0 -55.0 4375.415 0.0 -55.0 0.0 -10.175 55.0 -0.47781818181818186 55.0 0.0 55 -NULL -19 1969-12-31 16:00:08.607 -19.0 NULL 19 -19 0 -19.0 -0.0 19.0 -1511.507 0.0 19.0 0.0 -10.175 -19.0 1.3831578947368421 -19.0 0.0 -19 -NULL -49 1969-12-31 16:00:09.473 -49.0 NULL 49 -49 0 -49.0 -0.0 49.0 -3898.0969999999998 0.0 49.0 0.0 -10.175 -49.0 0.5363265306122449 -49.0 0.0 -49 -true 19 1969-12-31 16:00:03.827 19.0 cvLH6Eat2yFsyy7p -19 19 0 19.0 0.0 -19.0 1511.507 0.0 -19.0 0.0 -10.175 19.0 -1.3831578947368421 19.0 0.0 19 -NULL 13 1969-12-31 16:00:09.387 13.0 NULL -13 13 0 13.0 0.0 -13.0 1034.1889999999999 0.0 -13.0 0.0 -10.175 13.0 -2.0215384615384617 13.0 0.0 13 -NULL 42 1969-12-31 16:00:16.251 42.0 NULL -42 42 0 42.0 0.0 -42.0 3341.2259999999997 0.0 -42.0 0.0 -10.175 42.0 -0.6257142857142858 42.0 0.0 42 -NULL 11 1969-12-31 16:00:15.338 11.0 NULL -11 11 0 11.0 0.0 -11.0 875.083 0.0 -11.0 0.0 -10.175 11.0 -2.389090909090909 11.0 0.0 11 -NULL 39 1969-12-31 16:00:14.32 39.0 NULL -39 39 0 39.0 0.0 -39.0 3102.567 0.0 -39.0 0.0 -10.175 39.0 -0.6738461538461539 39.0 0.0 39 -NULL 9 1969-12-31 16:00:00.945 9.0 NULL -9 9 0 9.0 0.0 -9.0 715.977 0.0 -9.0 0.0 -10.175 9.0 -2.92 9.0 0.0 9 +NULL -26 1969-12-31 16:00:08.33 -26.0 NULL 26 -26 0 -26.0 -0.0 26.0 -2068.3779999999997 0.0 26.0 0.0 -10.175 -26.0 1.0107692307692309 -26.0 0.0 -26 +NULL -6 1969-12-31 16:00:07.423 -6.0 NULL 6 -6 0 -6.0 -0.0 6.0 -477.318 0.0 6.0 0.0 -10.175 -6.0 4.38 -6.0 0.0 -6 +true 55 1969-12-31 15:59:58.969 55.0 cvLH6Eat2yFsyy7p -55 55 0 55.0 0.0 -55.0 4375.415 0.0 -55.0 0.0 -10.175 55.0 -0.47781818181818186 55.0 0.0 55 +true 14 1969-12-31 16:00:08.209 14.0 1cGVWH7n1QU -14 14 0 14.0 0.0 -14.0 1113.742 0.0 -14.0 0.0 -10.175 14.0 -1.8771428571428572 14.0 0.0 14 +true 48 1969-12-31 16:00:00.923 48.0 1cGVWH7n1QU -48 48 0 48.0 0.0 -48.0 3818.544 0.0 -48.0 0.0 -10.175 48.0 -0.5475 48.0 0.0 48 +true -10 1969-12-31 16:00:00.082 -10.0 1cGVWH7n1QU 10 -10 0 -10.0 -0.0 10.0 -795.53 0.0 10.0 0.0 -10.175 -10.0 2.628 -10.0 0.0 -10 +true -55 1969-12-31 16:00:05.591 -55.0 1cGVWH7n1QU 55 -55 0 -55.0 -0.0 55.0 -4375.415 0.0 55.0 0.0 -10.175 -55.0 0.47781818181818186 -55.0 0.0 -55 +NULL -35 1969-12-31 16:00:04.52 -35.0 NULL 35 -35 0 -35.0 -0.0 35.0 -2784.355 0.0 35.0 0.0 -10.175 -35.0 0.7508571428571429 -35.0 0.0 -35 +NULL -8 1969-12-31 16:00:00.65 -8.0 NULL 8 -8 0 -8.0 -0.0 8.0 -636.424 0.0 8.0 0.0 -10.175 -8.0 3.285 -8.0 0.0 -8 +NULL -20 1969-12-31 16:00:13.131 -20.0 NULL 20 -20 0 -20.0 -0.0 20.0 -1591.06 0.0 20.0 0.0 -10.175 -20.0 1.314 -20.0 0.0 -20 +true 30 1969-12-31 16:00:15.705 30.0 821UdmGbkEf4j -30 30 0 30.0 0.0 -30.0 2386.59 0.0 -30.0 0.0 -10.175 30.0 -0.876 30.0 0.0 30 +true 40 1969-12-31 16:00:16.188 40.0 1cGVWH7n1QU -40 40 0 40.0 0.0 -40.0 3182.12 0.0 -40.0 0.0 -10.175 40.0 -0.657 40.0 0.0 40 +NULL -21 1969-12-31 16:00:12.544 -21.0 NULL 21 -21 0 -21.0 -0.0 21.0 -1670.6129999999998 0.0 21.0 0.0 -10.175 -21.0 1.2514285714285716 -21.0 0.0 -21 +true -34 1969-12-31 16:00:14.043 -34.0 821UdmGbkEf4j 34 -34 0 -34.0 -0.0 34.0 -2704.8019999999997 0.0 34.0 0.0 -10.175 -34.0 0.7729411764705882 -34.0 0.0 -34 +NULL 23 1969-12-31 16:00:08.307 23.0 NULL -23 23 0 23.0 0.0 -23.0 1829.719 0.0 -23.0 0.0 -10.175 23.0 -1.142608695652174 23.0 0.0 23 +true 8 1969-12-31 16:00:10.496 8.0 821UdmGbkEf4j -8 8 0 8.0 0.0 -8.0 636.424 0.0 -8.0 0.0 -10.175 8.0 -3.285 8.0 0.0 8 +NULL -55 1969-12-31 16:00:08.896 -55.0 NULL 55 -55 0 -55.0 -0.0 55.0 -4375.415 0.0 55.0 0.0 -10.175 -55.0 0.47781818181818186 -55.0 0.0 -55 +NULL -57 1969-12-31 16:00:02.707 -57.0 NULL 57 -57 0 -57.0 -0.0 57.0 -4534.521 0.0 57.0 0.0 -10.175 -57.0 0.4610526315789474 -57.0 0.0 -57 +true 1 1969-12-31 16:00:00.125 1.0 cvLH6Eat2yFsyy7p -1 1 0 1.0 0.0 -1.0 79.553 0.0 -1.0 0.0 -10.175 1.0 -26.28 1.0 0.0 1 +true -1 1969-12-31 16:00:00.107 -1.0 1cGVWH7n1QU 1 -1 0 -1.0 -0.0 1.0 -79.553 0.0 1.0 0.0 -10.175 -1.0 26.28 -1.0 0.0 -1 +true -59 1969-12-31 16:00:11.231 -59.0 821UdmGbkEf4j 59 -59 0 -59.0 -0.0 59.0 -4693.6269999999995 0.0 59.0 0.0 -10.175 -59.0 0.44542372881355935 -59.0 0.0 -59 +NULL 33 1969-12-31 16:00:07.006 33.0 NULL -33 33 0 33.0 0.0 -33.0 2625.249 0.0 -33.0 0.0 -10.175 33.0 -0.7963636363636364 33.0 0.0 33 +NULL -58 1969-12-31 16:00:15.658 -58.0 NULL 58 -58 0 -58.0 -0.0 58.0 -4614.074 0.0 58.0 0.0 -10.175 -58.0 0.4531034482758621 -58.0 0.0 -58 +true -6 1969-12-31 16:00:07.372 -6.0 cvLH6Eat2yFsyy7p 6 -6 0 -6.0 -0.0 6.0 -477.318 0.0 6.0 0.0 -10.175 -6.0 4.38 -6.0 0.0 -6 +true -27 1969-12-31 16:00:14.723 -27.0 1cGVWH7n1QU 27 -27 0 -27.0 -0.0 27.0 -2147.931 0.0 27.0 0.0 -10.175 -27.0 0.9733333333333334 -27.0 0.0 -27 +NULL 29 1969-12-31 16:00:16.19 29.0 NULL -29 29 0 29.0 0.0 -29.0 2307.037 0.0 -29.0 0.0 -10.175 29.0 -0.9062068965517242 29.0 0.0 29 +NULL -27 1969-12-31 16:00:15.435 -27.0 NULL 27 -27 0 -27.0 -0.0 27.0 -2147.931 0.0 27.0 0.0 -10.175 -27.0 0.9733333333333334 -27.0 0.0 -27 +true 33 1969-12-31 16:00:09.924 33.0 cvLH6Eat2yFsyy7p -33 33 0 33.0 0.0 -33.0 2625.249 0.0 -33.0 0.0 -10.175 33.0 -0.7963636363636364 33.0 0.0 33 +NULL -18 1969-12-31 16:00:00.564 -18.0 NULL 18 -18 0 -18.0 -0.0 18.0 -1431.954 0.0 18.0 0.0 -10.175 -18.0 1.46 -18.0 0.0 -18 +NULL -51 1969-12-31 16:00:02.011 -51.0 NULL 51 -51 0 -51.0 -0.0 51.0 -4057.203 0.0 51.0 0.0 -10.175 -51.0 0.5152941176470588 -51.0 0.0 -51 +true -8 1969-12-31 16:00:02.109 -8.0 cvLH6Eat2yFsyy7p 8 -8 0 -8.0 -0.0 8.0 -636.424 0.0 8.0 0.0 -10.175 -8.0 3.285 -8.0 0.0 -8 +true -13 1969-12-31 16:00:15.223 -13.0 1cGVWH7n1QU 13 -13 0 -13.0 -0.0 13.0 -1034.1889999999999 0.0 13.0 0.0 -10.175 -13.0 2.0215384615384617 -13.0 0.0 -13 +true -52 1969-12-31 16:00:01.824 -52.0 821UdmGbkEf4j 52 -52 0 -52.0 -0.0 52.0 -4136.755999999999 0.0 52.0 0.0 -10.175 -52.0 0.5053846153846154 -52.0 0.0 -52 +true -28 1969-12-31 16:00:10.554 -28.0 821UdmGbkEf4j 28 -28 0 -28.0 -0.0 28.0 -2227.484 0.0 28.0 0.0 -10.175 -28.0 0.9385714285714286 -28.0 0.0 -28 +true 2 1969-12-31 16:00:06.546 2.0 821UdmGbkEf4j -2 2 0 2.0 0.0 -2.0 159.106 0.0 -2.0 0.0 -10.175 2.0 -13.14 2.0 0.0 2 +true 40 1969-12-31 16:00:02.842 40.0 1cGVWH7n1QU -40 40 0 40.0 0.0 -40.0 3182.12 0.0 -40.0 0.0 -10.175 40.0 -0.657 40.0 0.0 40 +NULL 55 1969-12-31 16:00:02.215 55.0 NULL -55 55 0 55.0 0.0 -55.0 4375.415 0.0 -55.0 0.0 -10.175 55.0 -0.47781818181818186 55.0 0.0 55 +NULL 22 1969-12-31 16:00:00.621 22.0 NULL -22 22 0 22.0 0.0 -22.0 1750.166 0.0 -22.0 0.0 -10.175 22.0 -1.1945454545454546 22.0 0.0 22 +NULL 58 1969-12-31 16:00:03.366 58.0 NULL -58 58 0 58.0 0.0 -58.0 4614.074 0.0 -58.0 0.0 -10.175 58.0 -0.4531034482758621 58.0 0.0 58 +true 0 1969-12-31 16:00:08.742 0.0 821UdmGbkEf4j 0 0 0 0.0 0.0 -0.0 0.0 0.0 -0.0 0.0 -10.175 0.0 NULL 0.0 NULL 0 diff --git a/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out b/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out index 1d11b30..ee6672c 100644 --- a/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out +++ b/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out @@ -242,7 +242,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -306,7 +306,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) - auto parallelism: true + auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: @@ -528,7 +528,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 1298 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -591,7 +591,7 @@ STAGE PLANS: Map-reduce partition columns: p_partkey (type: int) Statistics: Num rows: 325 Data size: 1300 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -660,7 +660,7 @@ STAGE PLANS: Statistics: Num rows: 357 Data size: 1430 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - auto parallelism: true + auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: @@ -679,7 +679,7 @@ STAGE PLANS: Statistics: Num rows: 357 Data size: 1430 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) - auto parallelism: true + auto parallelism: false Reducer 4 Needs Tagging: false Reduce Operator Tree: @@ -836,7 +836,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1089,7 +1089,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1153,7 +1153,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) - auto parallelism: true + auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: @@ -1373,7 +1373,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1437,7 +1437,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) - auto parallelism: true + auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: @@ -1667,7 +1667,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1735,7 +1735,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: int) Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 - auto parallelism: true + auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: @@ -1751,7 +1751,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int) - auto parallelism: true + auto parallelism: false Execution mode: vectorized Reducer 4 Needs Tagging: false @@ -1930,7 +1930,7 @@ STAGE PLANS: Map-reduce partition columns: p_partkey (type: int) Statistics: Num rows: 325 Data size: 1300 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1990,7 +1990,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -2094,7 +2094,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 1298 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - auto parallelism: true + auto parallelism: false Stage: Stage-0 Fetch Operator @@ -2235,7 +2235,7 @@ STAGE PLANS: Map-reduce partition columns: p_partkey (type: int) Statistics: Num rows: 325 Data size: 1300 Basic stats: COMPLETE Column stats: NONE tag: 0 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -2295,7 +2295,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -2399,7 +2399,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 1298 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - auto parallelism: true + auto parallelism: false Stage: Stage-0 Fetch Operator @@ -2548,7 +2548,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -2612,7 +2612,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) - auto parallelism: true + auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: @@ -2821,7 +2821,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -2885,7 +2885,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) - auto parallelism: true + auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: @@ -3096,7 +3096,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -3160,7 +3160,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) - auto parallelism: true + auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: @@ -3381,7 +3381,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -3443,7 +3443,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: bigint), _col10 (type: string), _col11 (type: struct) - auto parallelism: true + auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: @@ -3462,7 +3462,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) - auto parallelism: true + auto parallelism: false Reducer 4 Needs Tagging: false Reduce Operator Tree: @@ -3703,7 +3703,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -3767,7 +3767,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) - auto parallelism: true + auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: @@ -4088,7 +4088,7 @@ STAGE PLANS: Map-reduce partition columns: p_partkey (type: int) Statistics: Num rows: 325 Data size: 1300 Basic stats: COMPLETE Column stats: NONE tag: 1 - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -4148,7 +4148,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -4212,7 +4212,7 @@ STAGE PLANS: Statistics: Num rows: 357 Data size: 1430 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) - auto parallelism: true + auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: @@ -4267,7 +4267,7 @@ STAGE PLANS: Statistics: Num rows: 2 Data size: 1298 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) - auto parallelism: true + auto parallelism: false Stage: Stage-0 Fetch Operator @@ -4403,7 +4403,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -4471,7 +4471,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: int) Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 - auto parallelism: true + auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: @@ -4527,31 +4527,31 @@ order by p_name) POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### -Manufacturer#3 almond antique forest lavender goldenrod 14 -Manufacturer#4 almond aquamarine floral ivory bisque 27 -Manufacturer#3 almond antique olive coral navajo 45 -Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 Manufacturer#4 almond antique gainsboro frosted violet 10 -Manufacturer#2 almond aquamarine rose maroon antique 25 -Manufacturer#5 almond aquamarine dodger light gainsboro 46 -Manufacturer#1 almond antique salmon chartreuse burlywood 6 -Manufacturer#1 almond antique burnished rose metallic 2 -Manufacturer#4 almond azure aquamarine papaya violet 12 -Manufacturer#1 almond aquamarine burnished black steel 28 +Manufacturer#5 almond azure blanched chiffon midnight 23 Manufacturer#2 almond antique violet turquoise frosted 40 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 +Manufacturer#1 almond aquamarine burnished black steel 28 +Manufacturer#2 almond aquamarine midnight light salmon 2 +Manufacturer#2 almond aquamarine rose maroon antique 25 +Manufacturer#3 almond antique misty red olive 1 +Manufacturer#4 almond aquamarine floral ivory bisque 27 Manufacturer#1 almond antique chartreuse lavender yellow 34 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 +Manufacturer#3 almond antique forest lavender goldenrod 14 +Manufacturer#5 almond antique medium spring khaki 6 +Manufacturer#3 almond antique metallic orange dim 19 Manufacturer#4 almond antique violet mint lemon 39 -Manufacturer#2 almond aquamarine midnight light salmon 2 -Manufacturer#5 almond azure blanched chiffon midnight 23 +Manufacturer#5 almond antique sky peru orange 2 +Manufacturer#4 almond azure aquamarine papaya violet 12 Manufacturer#2 almond antique violet chocolate turquoise 14 +Manufacturer#3 almond antique olive coral navajo 45 Manufacturer#4 almond aquamarine yellow dodger mint 7 -Manufacturer#1 almond aquamarine pink moccasin thistle 42 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 Manufacturer#5 almond antique blue firebrick mint 31 -Manufacturer#5 almond antique medium spring khaki 6 -Manufacturer#3 almond antique misty red olive 1 -Manufacturer#3 almond antique metallic orange dim 19 -Manufacturer#5 almond antique sky peru orange 2 Manufacturer#3 almond antique chartreuse khaki white 17 +Manufacturer#1 almond antique burnished rose metallic 2 PREHOOK: query: -- 16. testViewAsTableInputToPTF create view IF NOT EXISTS mfgr_price_view as select p_mfgr, p_brand, @@ -4681,7 +4681,7 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col2 (type: double) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -4748,7 +4748,7 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 1298 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double) - auto parallelism: true + auto parallelism: false Execution mode: vectorized Reducer 3 Needs Tagging: false @@ -4764,7 +4764,7 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 1298 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double) - auto parallelism: true + auto parallelism: false Reducer 4 Needs Tagging: false Reduce Operator Tree: @@ -5142,7 +5142,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -5202,7 +5202,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) - auto parallelism: true + auto parallelism: false Select Operator expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) outputColumnNames: _col1, _col2, _col5, _col7 @@ -5214,7 +5214,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) - auto parallelism: true + auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: @@ -5229,7 +5229,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _wcol0 (type: bigint), _col1 (type: string), _col2 (type: string), _col5 (type: int) - auto parallelism: true + auto parallelism: false Reducer 4 Needs Tagging: false Reduce Operator Tree: @@ -5648,7 +5648,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -5710,7 +5710,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: bigint), _col10 (type: string), _col11 (type: struct) - auto parallelism: true + auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: @@ -5729,7 +5729,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) - auto parallelism: true + auto parallelism: false Reducer 4 Needs Tagging: false Reduce Operator Tree: @@ -6001,7 +6001,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -6061,7 +6061,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: bigint), _col10 (type: string), _col11 (type: struct) - auto parallelism: true + auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: @@ -6076,7 +6076,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: bigint), _col10 (type: string), _col11 (type: struct) - auto parallelism: true + auto parallelism: false Reducer 4 Needs Tagging: false Reduce Operator Tree: @@ -6095,7 +6095,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) - auto parallelism: true + auto parallelism: false Reducer 5 Needs Tagging: false Reduce Operator Tree: @@ -6350,7 +6350,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -6410,7 +6410,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: bigint), _col10 (type: string), _col11 (type: struct) - auto parallelism: true + auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: @@ -6429,7 +6429,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) - auto parallelism: true + auto parallelism: false Reducer 4 Needs Tagging: false Reduce Operator Tree: @@ -6696,7 +6696,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -6756,7 +6756,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: bigint), _col10 (type: string), _col11 (type: struct) - auto parallelism: true + auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: @@ -6773,7 +6773,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: bigint), _col10 (type: string), _col11 (type: struct) - auto parallelism: true + auto parallelism: false Reducer 4 Needs Tagging: false Reduce Operator Tree: @@ -6792,7 +6792,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) - auto parallelism: true + auto parallelism: false Reducer 5 Needs Tagging: false Reduce Operator Tree: @@ -7090,7 +7090,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -7152,7 +7152,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: bigint), _col10 (type: string), _col11 (type: struct) - auto parallelism: true + auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: @@ -7171,7 +7171,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) - auto parallelism: true + auto parallelism: false Reducer 4 Needs Tagging: false Reduce Operator Tree: @@ -7435,7 +7435,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) - auto parallelism: true + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -7497,7 +7497,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: bigint), _col10 (type: string), _col11 (type: struct) - auto parallelism: true + auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: @@ -7516,7 +7516,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 2597 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) - auto parallelism: true + auto parallelism: false Reducer 4 Needs Tagging: false Reduce Operator Tree: diff --git a/ql/src/test/results/clientpositive/spark/windowing.q.out b/ql/src/test/results/clientpositive/spark/windowing.q.out index 424c444..5379b99 100644 --- a/ql/src/test/results/clientpositive/spark/windowing.q.out +++ b/ql/src/test/results/clientpositive/spark/windowing.q.out @@ -2365,3 +2365,15 @@ POSTHOOK: Input: default@part 1602.59 1549.8900000000003 4649.670000000001 1414.42 1523.5400000000004 3047.080000000001 1632.66 1632.6600000000008 1632.6600000000008 +PREHOOK: query: -- 47. empty partition +select sum(p_size) over (partition by p_mfgr ) +from part where p_mfgr = 'm1' +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 47. empty partition +select sum(p_size) over (partition by p_mfgr ) +from part where p_mfgr = 'm1' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/subquery_exists.q.out b/ql/src/test/results/clientpositive/subquery_exists.q.out index b7f4cb9..f939c7d 100644 --- a/ql/src/test/results/clientpositive/subquery_exists.q.out +++ b/ql/src/test/results/clientpositive/subquery_exists.q.out @@ -41,22 +41,22 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((value > 'val_9') and value is not null) and key is not null) (type: boolean) - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + predicate: ((value > 'val_9') and key is not null) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: string), key (type: string) outputColumnNames: _col1, _col2 - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col1 (type: string), _col2 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: diff --git a/ql/src/test/results/clientpositive/subquery_in.q.out b/ql/src/test/results/clientpositive/subquery_in.q.out index 769beda..84d316f 100644 --- a/ql/src/test/results/clientpositive/subquery_in.q.out +++ b/ql/src/test/results/clientpositive/subquery_in.q.out @@ -122,22 +122,22 @@ STAGE PLANS: alias: s1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > '9') and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key > '9') (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE @@ -243,22 +243,22 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key > '9') and key is not null) and value is not null) (type: boolean) - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > '9') and value is not null) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -686,37 +686,37 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key > '9') and key is not null) and value is not null) (type: boolean) - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > '9') and value is not null) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: key, value - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: key (type: string), value (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 21 Data size: 223 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 21 Data size: 223 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 21 Data size: 223 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -743,7 +743,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 21 Data size: 223 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: diff --git a/ql/src/test/results/clientpositive/subquery_in_having.q.out b/ql/src/test/results/clientpositive/subquery_in_having.q.out index ca3513e..163b0d1 100644 --- a/ql/src/test/results/clientpositive/subquery_in_having.q.out +++ b/ql/src/test/results/clientpositive/subquery_in_having.q.out @@ -802,22 +802,22 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > '8') and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key > '8') (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -1150,17 +1150,17 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > '8') and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key > '8') (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: 0 {key} {value} diff --git a/ql/src/test/results/clientpositive/subquery_multiinsert.q.out b/ql/src/test/results/clientpositive/subquery_multiinsert.q.out index ebd47ca..9d68139 100644 --- a/ql/src/test/results/clientpositive/subquery_multiinsert.q.out +++ b/ql/src/test/results/clientpositive/subquery_multiinsert.q.out @@ -247,22 +247,22 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key > '9') and key is not null) and value is not null) (type: boolean) - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > '9') and value is not null) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -490,7 +490,7 @@ POSTHOOK: Input: default@src_5 199 val_199 199 val_199 2 val_2 -Warning: Map Join MAPJOIN[109][bigTable=b] in task 'Stage-13:MAPRED' is a cross product +Warning: Map Join MAPJOIN[107][bigTable=b] in task 'Stage-13:MAPRED' is a cross product Warning: Shuffle Join JOIN[31][tables = [b, sq_2_notin_nullcheck]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: explain from src b @@ -643,17 +643,17 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key > '9') and key is not null) and value is not null) (type: boolean) - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > '9') and value is not null) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: 0 {key} {value} @@ -823,7 +823,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe -Warning: Map Join MAPJOIN[109][bigTable=b] in task 'Stage-13:MAPRED' is a cross product +Warning: Map Join MAPJOIN[107][bigTable=b] in task 'Stage-13:MAPRED' is a cross product Warning: Shuffle Join JOIN[31][tables = [b, sq_2_notin_nullcheck]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: from src b INSERT OVERWRITE TABLE src_4 diff --git a/ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out b/ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out index 273037a..027c195 100644 --- a/ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out +++ b/ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out @@ -86,22 +86,22 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key > '9') and key is not null) and value is not null) (type: boolean) - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > '9') and value is not null) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE TableScan alias: src11 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE @@ -121,14 +121,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {KEY.reducesinkkey1} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 46 Data size: 490 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 46 Data size: 490 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 46 Data size: 490 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -167,22 +167,22 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key > '9') and key is not null) and value is not null) (type: boolean) - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > '9') and value is not null) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -533,37 +533,37 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key > '9') and key is not null) and value is not null) (type: boolean) - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > '9') and value is not null) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: key, value - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: key (type: string), value (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 21 Data size: 223 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 21 Data size: 223 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 21 Data size: 223 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -590,7 +590,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 21 Data size: 223 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: diff --git a/ql/src/test/results/clientpositive/subquery_views.q.out b/ql/src/test/results/clientpositive/subquery_views.q.out index 50a5e29..c928db4 100644 --- a/ql/src/test/results/clientpositive/subquery_views.q.out +++ b/ql/src/test/results/clientpositive/subquery_views.q.out @@ -146,11 +146,11 @@ STAGE PLANS: alias: b Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < '11') and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key < '11') (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE value expressions: key (type: string), value (type: string) Reduce Operator Tree: Join Operator @@ -160,7 +160,7 @@ STAGE PLANS: 0 {VALUE._col0} {VALUE._col1} 1 outputColumnNames: _col0, _col1 - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -176,7 +176,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string), _col0 (type: string) sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col0 (type: string) - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE @@ -200,19 +200,19 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {KEY.reducesinkkey0} outputColumnNames: _col0, _col5 - Statistics: Num rows: 100 Data size: 1065 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 200 Data size: 2132 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col5 is null (type: boolean) - Statistics: Num rows: 50 Data size: 532 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 100 Data size: 1066 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 50 Data size: 532 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 100 Data size: 1066 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 50 Data size: 532 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 100 Data size: 1066 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -235,7 +235,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 50 Data size: 532 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 100 Data size: 1066 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: diff --git a/ql/src/test/results/clientpositive/sum_expr_with_order.q.out b/ql/src/test/results/clientpositive/sum_expr_with_order.q.out new file mode 100644 index 0000000..00318e8 --- /dev/null +++ b/ql/src/test/results/clientpositive/sum_expr_with_order.q.out @@ -0,0 +1,15 @@ +PREHOOK: query: select +cast(sum(key)*100 as decimal(15,3)) as c1 +from src +order by c1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select +cast(sum(key)*100 as decimal(15,3)) as c1 +from src +order by c1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +13009100 diff --git a/ql/src/test/results/clientpositive/tez/acid_vectorization.q.out b/ql/src/test/results/clientpositive/tez/acid_vectorization.q.out new file mode 100644 index 0000000..1792979 --- /dev/null +++ b/ql/src/test/results/clientpositive/tez/acid_vectorization.q.out @@ -0,0 +1,62 @@ +PREHOOK: query: CREATE TABLE acid_vectorized(a INT, b STRING) CLUSTERED BY(a) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_vectorized +POSTHOOK: query: CREATE TABLE acid_vectorized(a INT, b STRING) CLUSTERED BY(a) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_vectorized +PREHOOK: query: insert into table acid_vectorized select cint, cstring1 from alltypesorc where cint is not null order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_vectorized +POSTHOOK: query: insert into table acid_vectorized select cint, cstring1 from alltypesorc where cint is not null order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_vectorized +POSTHOOK: Lineage: acid_vectorized.a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_vectorized.b SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: insert into table acid_vectorized values (1, 'bar') +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@acid_vectorized +POSTHOOK: query: insert into table acid_vectorized values (1, 'bar') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@acid_vectorized +POSTHOOK: Lineage: acid_vectorized.a EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: acid_vectorized.b SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: update acid_vectorized set b = 'foo' where b = 'bar' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_vectorized +PREHOOK: Output: default@acid_vectorized +POSTHOOK: query: update acid_vectorized set b = 'foo' where b = 'bar' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_vectorized +POSTHOOK: Output: default@acid_vectorized +PREHOOK: query: delete from acid_vectorized where b = 'foo' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_vectorized +PREHOOK: Output: default@acid_vectorized +POSTHOOK: query: delete from acid_vectorized where b = 'foo' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_vectorized +POSTHOOK: Output: default@acid_vectorized +PREHOOK: query: select a, b from acid_vectorized order by a, b +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_vectorized +#### A masked pattern was here #### +POSTHOOK: query: select a, b from acid_vectorized order by a, b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_vectorized +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa +-1073051226 A34p7oRr2WvUJNf +-1072910839 0iqrc5 +-1072081801 dPkN74F7 +-1072076362 2uLyD28144vklju213J1mr +-1071480828 aw724t8c5558x2xneC624 +-1071363017 Anj0oF +-1070883071 0ruyd6Y50JpdGRf6HqD +-1070551679 iUR3Q +-1069736047 k17Am8uPHWk02cEf1jet diff --git a/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_10.q.out b/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_10.q.out index d661eef..3872291 100644 --- a/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_10.q.out +++ b/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_10.q.out @@ -82,7 +82,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) + predicate: (key < 6) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -113,7 +113,7 @@ STAGE PLANS: TableScan alias: a Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) + predicate: (key < 6) (type: boolean) Select Operator expressions: key (type: int) outputColumnNames: _col0 @@ -145,22 +145,22 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Reducer 3 Reduce Operator Tree: Group By Operator @@ -252,24 +252,24 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: key - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() bucketGroup: true keys: key (type: int) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Map 3 Map Operator Tree: @@ -277,12 +277,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -294,9 +294,9 @@ STAGE PLANS: 1 _col0 (type: int) input vertices: 0 Reducer 2 - Statistics: Num rows: 2 Data size: 15 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 2 Data size: 15 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash diff --git a/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_5.q.out b/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_5.q.out index d238592..b42bac7 100644 --- a/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_5.q.out +++ b/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_5.q.out @@ -110,70 +110,41 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan - alias: b - Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + alias: a + Statistics: Num rows: 2 Data size: 226 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 14 Data size: 1425 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 - Estimated key counts: Map 3 => 1 - keys: - 0 key (type: string) - 1 key (type: string) - input vertices: - 0 Map 3 - Position of Big Table: 1 - Statistics: Num rows: 15 Data size: 1567 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 15 Data size: 1567 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col0 (type: bigint) - auto parallelism: false + Statistics: Num rows: 1 Data size: 113 Basic stats: COMPLETE Column stats: NONE Path -> Alias: #### A masked pattern was here #### Path -> Partition: #### A masked pattern was here #### Partition - base file name: bucket_big input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: COLUMN_STATS_ACCURATE true SORTBUCKETCOLSPREFIX TRUE - bucket_count 2 + bucket_count 4 bucket_field_name key columns key,value columns.comments columns.types string:string #### A masked pattern was here #### - name default.bucket_big - numFiles 2 - serialization.ddl struct bucket_big { string key, string value} + name default.bucket_small + numFiles 4 + serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 2750 + totalSize 226 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -182,64 +153,79 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE true SORTBUCKETCOLSPREFIX TRUE - bucket_count 2 + bucket_count 4 bucket_field_name key columns key,value columns.comments columns.types string:string #### A masked pattern was here #### - name default.bucket_big - numFiles 2 - serialization.ddl struct bucket_big { string key, string value} + name default.bucket_small + numFiles 4 + serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 2750 + totalSize 226 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_big - name: default.bucket_big + name: default.bucket_small + name: default.bucket_small Truncated Path -> Alias: - /bucket_big [b] - Map 3 + /bucket_small [a] Map Operator Tree: TableScan - alias: a - Statistics: Num rows: 2 Data size: 226 Basic stats: COMPLETE Column stats: NONE + alias: b + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 113 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 1 Data size: 113 Basic stats: COMPLETE Column stats: NONE - tag: 0 - auto parallelism: true + Statistics: Num rows: 14 Data size: 1425 Basic stats: COMPLETE Column stats: NONE + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Statistics: Num rows: 15 Data size: 1567 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 15 Data size: 1567 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: bigint) + auto parallelism: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: #### A masked pattern was here #### Partition - base file name: bucket_small input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: COLUMN_STATS_ACCURATE true SORTBUCKETCOLSPREFIX TRUE - bucket_count 4 + bucket_count 2 bucket_field_name key columns key,value columns.comments columns.types string:string #### A masked pattern was here #### - name default.bucket_small - numFiles 4 - serialization.ddl struct bucket_small { string key, string value} + name default.bucket_big + numFiles 2 + serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 226 + totalSize 2750 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -248,24 +234,24 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE true SORTBUCKETCOLSPREFIX TRUE - bucket_count 4 + bucket_count 2 bucket_field_name key columns key,value columns.comments columns.types string:string #### A masked pattern was here #### - name default.bucket_small - numFiles 4 - serialization.ddl struct bucket_small { string key, string value} + name default.bucket_big + numFiles 2 + serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 226 + totalSize 2750 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_small - name: default.bucket_small + name: default.bucket_big + name: default.bucket_big Truncated Path -> Alias: - /bucket_small [a] + /bucket_big [b] Reducer 2 Needs Tagging: false Reduce Operator Tree: @@ -361,11 +347,10 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) Reducer 3 <- Map 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Map 2 Map Operator Tree: TableScan alias: b @@ -375,19 +360,11 @@ STAGE PLANS: isSamplingPred: false predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 113 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 1 Data size: 113 Basic stats: COMPLETE Column stats: NONE - tag: 1 - auto parallelism: true Path -> Alias: #### A masked pattern was here #### Path -> Partition: #### A masked pattern was here #### Partition - base file name: bucket_small input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: @@ -431,7 +408,6 @@ STAGE PLANS: name: default.bucket_small Truncated Path -> Alias: /bucket_small [b] - Map 2 Map Operator Tree: TableScan alias: a @@ -441,18 +417,15 @@ STAGE PLANS: isSamplingPred: false predicate: key is not null (type: boolean) Statistics: Num rows: 14 Data size: 1425 Basic stats: COMPLETE Column stats: NONE - Map Join Operator + Merge Join Operator condition map: Inner Join 0 to 1 condition expressions: 0 1 - Estimated key counts: Map 1 => 1 keys: 0 key (type: string) 1 key (type: string) - input vertices: - 1 Map 1 Position of Big Table: 0 Statistics: Num rows: 15 Data size: 1567 Basic stats: COMPLETE Column stats: NONE Select Operator @@ -473,7 +446,6 @@ STAGE PLANS: Path -> Partition: #### A masked pattern was here #### Partition - base file name: bucket_big input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: diff --git a/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_9.q.out b/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_9.q.out index 571ea0a..fb44d5f 100644 --- a/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_9.q.out +++ b/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_9.q.out @@ -725,12 +725,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -742,9 +742,9 @@ STAGE PLANS: 1 _col0 (type: int) input vertices: 1 Map 3 - Statistics: Num rows: 2 Data size: 15 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 2 Data size: 15 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -760,17 +760,17 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Group By Operator @@ -1167,12 +1167,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 8) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 8) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -1184,9 +1184,9 @@ STAGE PLANS: 1 _col0 (type: int) input vertices: 1 Map 3 - Statistics: Num rows: 2 Data size: 15 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 2 Data size: 15 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -1202,17 +1202,17 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 8) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 8) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Group By Operator @@ -1415,17 +1415,17 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Map 2 Map Operator Tree: TableScan @@ -1531,17 +1531,17 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Map 2 Map Operator Tree: TableScan @@ -1657,29 +1657,29 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Map 2 Map Operator Tree: TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -1695,9 +1695,9 @@ STAGE PLANS: input vertices: 1 Map 4 2 Map 1 - Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 46 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 46 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -1713,17 +1713,17 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Reducer 3 Reduce Operator Tree: Group By Operator @@ -2603,12 +2603,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -2620,9 +2620,9 @@ STAGE PLANS: 1 _col0 (type: int) input vertices: 1 Map 3 - Statistics: Num rows: 2 Data size: 15 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 2 Data size: 15 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -2638,17 +2638,17 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Group By Operator @@ -3045,12 +3045,12 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 8) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 8) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -3062,9 +3062,9 @@ STAGE PLANS: 1 _col0 (type: int) input vertices: 1 Map 3 - Statistics: Num rows: 2 Data size: 15 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 2 Data size: 15 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -3080,17 +3080,17 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 8) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 8) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Group By Operator @@ -3167,17 +3167,17 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Map 2 Map Operator Tree: TableScan @@ -3283,17 +3283,17 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Map 2 Map Operator Tree: TableScan @@ -3409,29 +3409,29 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Map 2 Map Operator Tree: TableScan alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -3447,9 +3447,9 @@ STAGE PLANS: input vertices: 1 Map 4 2 Map 1 - Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 46 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 46 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -3465,17 +3465,17 @@ STAGE PLANS: alias: a Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 6) and key is not null) (type: boolean) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 6) (type: boolean) + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Reducer 3 Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/tez/bucket3.q.out b/ql/src/test/results/clientpositive/tez/bucket3.q.out index 2b19e16..17e1985 100644 --- a/ql/src/test/results/clientpositive/tez/bucket3.q.out +++ b/ql/src/test/results/clientpositive/tez/bucket3.q.out @@ -235,7 +235,7 @@ STAGE PLANS: Filter Operator predicate: (((hash(key) & 2147483647) % 2) = 0) (type: boolean) Select Operator - expressions: key (type: int), value (type: string), ds (type: string) + expressions: key (type: int), value (type: string), '1' (type: string) outputColumnNames: _col0, _col1, _col2 ListSink diff --git a/ql/src/test/results/clientpositive/tez/bucket_map_join_tez2.q.out b/ql/src/test/results/clientpositive/tez/bucket_map_join_tez2.q.out index a538839..240decc 100644 --- a/ql/src/test/results/clientpositive/tez/bucket_map_join_tez2.q.out +++ b/ql/src/test/results/clientpositive/tez/bucket_map_join_tez2.q.out @@ -320,12 +320,12 @@ STAGE PLANS: alias: tab_part Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > 1) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key > 1) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -338,14 +338,14 @@ STAGE PLANS: outputColumnNames: _col0, _col1 input vertices: 1 Map 2 - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -356,17 +356,17 @@ STAGE PLANS: alias: tab_part Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > 2) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key > 2) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/tez/cbo_correctness.q.out b/ql/src/test/results/clientpositive/tez/cbo_correctness.q.out index 5920612..2103d08 100644 --- a/ql/src/test/results/clientpositive/tez/cbo_correctness.q.out +++ b/ql/src/test/results/clientpositive/tez/cbo_correctness.q.out @@ -16719,7 +16719,7 @@ POSTHOOK: type: DROPVIEW POSTHOOK: Input: default@v4 POSTHOOK: Output: default@v4 PREHOOK: query: -- 11. Union All -select * from t1 union all select * from t2 order by key +select * from t1 union all select * from t2 order by key, c_boolean, value, dt PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t1@dt=2014 @@ -16727,7 +16727,7 @@ PREHOOK: Input: default@t2 PREHOOK: Input: default@t2@dt=2014 #### A masked pattern was here #### POSTHOOK: query: -- 11. Union All -select * from t1 union all select * from t2 order by key +select * from t1 union all select * from t2 order by key, c_boolean, value, dt POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t1@dt=2014 @@ -16758,13 +16758,13 @@ null null NULL NULL NULL 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 -1 1 1 1.0 true 2014 1 1 1 1.0 false 2014 -1 1 1 1.0 true 2014 1 1 1 1.0 false 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 +1 1 1 1.0 true 2014 +1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 2 2 2 2.0 true 2014 @@ -18142,7 +18142,7 @@ from src_cbo where src_cbo.key not in ( select key from src_cbo s1 where s1.key > '2' - ) + ) order by key PREHOOK: type: QUERY PREHOOK: Input: default@src_cbo #### A masked pattern was here #### @@ -18153,129 +18153,129 @@ from src_cbo where src_cbo.key not in ( select key from src_cbo s1 where s1.key > '2' - ) + ) order by key POSTHOOK: type: QUERY POSTHOOK: Input: default@src_cbo #### A masked pattern was here #### -2 val_2 -199 val_199 -199 val_199 -199 val_199 -197 val_197 -197 val_197 -196 val_196 -195 val_195 -195 val_195 -194 val_194 -193 val_193 -193 val_193 -193 val_193 -192 val_192 -191 val_191 -191 val_191 -190 val_190 -19 val_19 -189 val_189 -187 val_187 -187 val_187 -187 val_187 -186 val_186 -183 val_183 -181 val_181 -180 val_180 -18 val_18 -18 val_18 -179 val_179 -179 val_179 -178 val_178 -177 val_177 -176 val_176 -176 val_176 -175 val_175 -175 val_175 -174 val_174 -174 val_174 -172 val_172 -172 val_172 -170 val_170 -17 val_17 -169 val_169 -169 val_169 -169 val_169 -169 val_169 -168 val_168 -167 val_167 -167 val_167 -167 val_167 -166 val_166 -165 val_165 -165 val_165 -164 val_164 -164 val_164 -163 val_163 -162 val_162 -160 val_160 -158 val_158 -157 val_157 -156 val_156 -155 val_155 -153 val_153 -152 val_152 -152 val_152 -150 val_150 -15 val_15 -15 val_15 -149 val_149 -149 val_149 -146 val_146 -146 val_146 -145 val_145 -143 val_143 +0 val_0 +0 val_0 +0 val_0 +10 val_10 +100 val_100 +100 val_100 +103 val_103 +103 val_103 +104 val_104 +104 val_104 +105 val_105 +11 val_11 +111 val_111 +113 val_113 +113 val_113 +114 val_114 +116 val_116 +118 val_118 +118 val_118 +119 val_119 +119 val_119 +119 val_119 +12 val_12 +12 val_12 +120 val_120 +120 val_120 +125 val_125 +125 val_125 +126 val_126 +128 val_128 +128 val_128 +128 val_128 +129 val_129 +129 val_129 +131 val_131 +133 val_133 +134 val_134 +134 val_134 +136 val_136 +137 val_137 +137 val_137 138 val_138 138 val_138 138 val_138 138 val_138 -137 val_137 -137 val_137 -136 val_136 -134 val_134 -134 val_134 -133 val_133 -131 val_131 -129 val_129 -129 val_129 -128 val_128 -128 val_128 -128 val_128 -126 val_126 -125 val_125 -125 val_125 -120 val_120 -120 val_120 -12 val_12 -12 val_12 -119 val_119 -119 val_119 -119 val_119 -118 val_118 -118 val_118 -116 val_116 -114 val_114 -113 val_113 -113 val_113 -111 val_111 -11 val_11 -105 val_105 -104 val_104 -104 val_104 -103 val_103 -103 val_103 -100 val_100 -100 val_100 -10 val_10 -0 val_0 -0 val_0 -0 val_0 +143 val_143 +145 val_145 +146 val_146 +146 val_146 +149 val_149 +149 val_149 +15 val_15 +15 val_15 +150 val_150 +152 val_152 +152 val_152 +153 val_153 +155 val_155 +156 val_156 +157 val_157 +158 val_158 +160 val_160 +162 val_162 +163 val_163 +164 val_164 +164 val_164 +165 val_165 +165 val_165 +166 val_166 +167 val_167 +167 val_167 +167 val_167 +168 val_168 +169 val_169 +169 val_169 +169 val_169 +169 val_169 +17 val_17 +170 val_170 +172 val_172 +172 val_172 +174 val_174 +174 val_174 +175 val_175 +175 val_175 +176 val_176 +176 val_176 +177 val_177 +178 val_178 +179 val_179 +179 val_179 +18 val_18 +18 val_18 +180 val_180 +181 val_181 +183 val_183 +186 val_186 +187 val_187 +187 val_187 +187 val_187 +189 val_189 +19 val_19 +190 val_190 +191 val_191 +191 val_191 +192 val_192 +193 val_193 +193 val_193 +193 val_193 +194 val_194 +195 val_195 +195 val_195 +196 val_196 +197 val_197 +197 val_197 +199 val_199 +199 val_199 +199 val_199 +2 val_2 PREHOOK: query: -- non agg, corr select p_mfgr, b.p_name, p_size from part b @@ -18946,17 +18946,162 @@ POSTHOOK: Input: default@src_cbo 96 1 97 2 98 2 -PREHOOK: query: -- 17. get stats with empty partition list +PREHOOK: query: -- 20. Test get stats with empty partition list select t1.value from t1 join t2 on t1.key = t2.key where t1.dt = '10' and t1.c_boolean = true PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 PREHOOK: Input: default@t2@dt=2014 #### A masked pattern was here #### -POSTHOOK: query: -- 17. get stats with empty partition list +POSTHOOK: query: -- 20. Test get stats with empty partition list select t1.value from t1 join t2 on t1.key = t2.key where t1.dt = '10' and t1.c_boolean = true POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t2 POSTHOOK: Input: default@t2@dt=2014 #### A masked pattern was here #### +PREHOOK: query: -- 21. Test groupby is empty and there is no other cols in aggr +select unionsrc.key FROM (select 'tst1' as key, count(1) as value from src) unionsrc +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: -- 21. Test groupby is empty and there is no other cols in aggr +select unionsrc.key FROM (select 'tst1' as key, count(1) as value from src) unionsrc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +tst1 +PREHOOK: query: select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src) unionsrc +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src) unionsrc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +tst1 500 +PREHOOK: query: select unionsrc.key FROM (select 'max' as key, max(c_int) as value from t3 s1 + UNION ALL + select 'min' as key, min(c_int) as value from t3 s2 + UNION ALL + select 'avg' as key, avg(c_int) as value from t3 s3) unionsrc order by unionsrc.key +PREHOOK: type: QUERY +PREHOOK: Input: default@t3 +#### A masked pattern was here #### +POSTHOOK: query: select unionsrc.key FROM (select 'max' as key, max(c_int) as value from t3 s1 + UNION ALL + select 'min' as key, min(c_int) as value from t3 s2 + UNION ALL + select 'avg' as key, avg(c_int) as value from t3 s3) unionsrc order by unionsrc.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t3 +#### A masked pattern was here #### +avg +max +min +PREHOOK: query: select unionsrc.key, unionsrc.value FROM (select 'max' as key, max(c_int) as value from t3 s1 + UNION ALL + select 'min' as key, min(c_int) as value from t3 s2 + UNION ALL + select 'avg' as key, avg(c_int) as value from t3 s3) unionsrc order by unionsrc.key +PREHOOK: type: QUERY +PREHOOK: Input: default@t3 +#### A masked pattern was here #### +POSTHOOK: query: select unionsrc.key, unionsrc.value FROM (select 'max' as key, max(c_int) as value from t3 s1 + UNION ALL + select 'min' as key, min(c_int) as value from t3 s2 + UNION ALL + select 'avg' as key, avg(c_int) as value from t3 s3) unionsrc order by unionsrc.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t3 +#### A masked pattern was here #### +avg 1.5 +max 3.0 +min 1.0 +PREHOOK: query: select unionsrc.key, count(1) FROM (select 'max' as key, max(c_int) as value from t3 s1 + UNION ALL + select 'min' as key, min(c_int) as value from t3 s2 + UNION ALL + select 'avg' as key, avg(c_int) as value from t3 s3) unionsrc group by unionsrc.key order by unionsrc.key +PREHOOK: type: QUERY +PREHOOK: Input: default@t3 +#### A masked pattern was here #### +POSTHOOK: query: select unionsrc.key, count(1) FROM (select 'max' as key, max(c_int) as value from t3 s1 + UNION ALL + select 'min' as key, min(c_int) as value from t3 s2 + UNION ALL + select 'avg' as key, avg(c_int) as value from t3 s3) unionsrc group by unionsrc.key order by unionsrc.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t3 +#### A masked pattern was here #### +avg 1 +max 1 +min 1 +PREHOOK: query: -- Windowing +select *, rank() over(partition by key order by value) as rr from src1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: -- Windowing +select *, rank() over(partition by key order by value) as rr from src1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### + 1 + 1 + 1 + 1 + val_165 5 + val_193 6 + val_265 7 + val_27 8 + val_409 9 + val_484 10 +128 1 +146 val_146 1 +150 val_150 1 +213 val_213 1 +224 1 +238 val_238 1 +255 val_255 1 +273 val_273 1 +278 val_278 1 +311 val_311 1 +369 1 +401 val_401 1 +406 val_406 1 +66 val_66 1 +98 val_98 1 +PREHOOK: query: select *, rank() over(partition by key order by value) from src1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: select *, rank() over(partition by key order by value) from src1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### + 1 + 1 + 1 + 1 + val_165 5 + val_193 6 + val_265 7 + val_27 8 + val_409 9 + val_484 10 +128 1 +146 val_146 1 +150 val_150 1 +213 val_213 1 +224 1 +238 val_238 1 +255 val_255 1 +273 val_273 1 +278 val_278 1 +311 val_311 1 +369 1 +401 val_401 1 +406 val_406 1 +66 val_66 1 +98 val_98 1 diff --git a/ql/src/test/results/clientpositive/tez/cross_product_check_1.q.out b/ql/src/test/results/clientpositive/tez/cross_product_check_1.q.out index 5286cac..cc13e37 100644 --- a/ql/src/test/results/clientpositive/tez/cross_product_check_1.q.out +++ b/ql/src/test/results/clientpositive/tez/cross_product_check_1.q.out @@ -182,7 +182,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[25][tables = [a, od1]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[24][tables = [a, od1]] in Stage 'Reducer 4' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 on d1.key = d2.key @@ -308,8 +308,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[18][tables = [d1, d2]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[19][tables = [a, od1]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[17][tables = [d1, d2]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[18][tables = [a, od1]] in Stage 'Reducer 4' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 PREHOOK: type: QUERY POSTHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 @@ -420,7 +420,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[30][tables = [ss, od1]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[29][tables = [ss, od1]] in Stage 'Reducer 4' is a cross product PREHOOK: query: explain select * from (select A.key from A group by key) ss join (select d1.key from B d1 join B d2 on d1.key = d2.key where 1 = 1 group by d1.key) od1 diff --git a/ql/src/test/results/clientpositive/tez/cross_product_check_2.q.out b/ql/src/test/results/clientpositive/tez/cross_product_check_2.q.out index 3a2d725..69c3001 100644 --- a/ql/src/test/results/clientpositive/tez/cross_product_check_2.q.out +++ b/ql/src/test/results/clientpositive/tez/cross_product_check_2.q.out @@ -177,7 +177,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[25][bigTable=a] in task 'Map 4' is a cross product +Warning: Map Join MAPJOIN[24][bigTable=a] in task 'Map 4' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 on d1.key = d2.key @@ -300,8 +300,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[18][bigTable=d1] in task 'Map 1' is a cross product -Warning: Map Join MAPJOIN[19][bigTable=a] in task 'Map 4' is a cross product +Warning: Map Join MAPJOIN[17][bigTable=d1] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[18][bigTable=a] in task 'Map 4' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 PREHOOK: type: QUERY POSTHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 @@ -410,7 +410,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[30][bigTable=?] in task 'Reducer 5' is a cross product +Warning: Map Join MAPJOIN[29][bigTable=?] in task 'Reducer 5' is a cross product PREHOOK: query: explain select * from (select A.key from A group by key) ss join (select d1.key from B d1 join B d2 on d1.key = d2.key where 1 = 1 group by d1.key) od1 diff --git a/ql/src/test/results/clientpositive/tez/delete_all_partitioned.q.out b/ql/src/test/results/clientpositive/tez/delete_all_partitioned.q.out index c5149b2..90f8753 100644 --- a/ql/src/test/results/clientpositive/tez/delete_all_partitioned.q.out +++ b/ql/src/test/results/clientpositive/tez/delete_all_partitioned.q.out @@ -84,5 +84,3 @@ POSTHOOK: Input: default@acid_dap POSTHOOK: Input: default@acid_dap@ds=today POSTHOOK: Input: default@acid_dap@ds=tomorrow #### A masked pattern was here #### --1071480828 aw724t8c5558x2xneC624 today --1072076362 2uLyD28144vklju213J1mr today diff --git a/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out b/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out index c5effa0..9a3315f 100644 --- a/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out +++ b/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out @@ -1900,10 +1900,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (ds is not null and (ds = '2008-04-08')) (type: boolean) + filterExpr: (ds = '2008-04-08') (type: boolean) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: ds (type: string) + expressions: '2008-04-08' (type: string) outputColumnNames: ds Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -2565,7 +2565,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: ds (type: string) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - value expressions: hr (type: string) Map 5 Map Operator Tree: TableScan @@ -2629,14 +2628,12 @@ STAGE PLANS: condition map: Inner Join 0 to 1 condition expressions: - 0 {VALUE._col2} + 0 1 - outputColumnNames: _col3 Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col3 (type: string) + key expressions: '11' (type: string) sort order: + - Map-reduce partition columns: _col3 (type: string) Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE Reducer 3 Reduce Operator Tree: @@ -2726,15 +2723,15 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart_hour - filterExpr: (hr is not null and (hr = 13)) (type: boolean) + filterExpr: (hr = 13) (type: boolean) Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (hr is not null and (hr = 13)) (type: boolean) - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + predicate: (hr = 13) (type: boolean) + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: '13' (type: string) sort order: + - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Map 6 Map Operator Tree: TableScan @@ -2770,9 +2767,9 @@ STAGE PLANS: condition expressions: 0 1 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -4360,10 +4357,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (ds is not null and (ds = '2008-04-08')) (type: boolean) + filterExpr: (ds = '2008-04-08') (type: boolean) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: ds (type: string) + expressions: '2008-04-08' (type: string) outputColumnNames: ds Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -4749,12 +4746,11 @@ STAGE PLANS: condition map: Inner Join 0 to 1 condition expressions: - 0 {hr} + 0 1 keys: 0 ds (type: string) 1 ds (type: string) - outputColumnNames: _col3 input vertices: 1 Map 4 Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE @@ -4765,7 +4761,7 @@ STAGE PLANS: 0 1 keys: - 0 _col3 (type: string) + 0 '11' (type: string) 1 '11' (type: string) input vertices: 1 Map 3 @@ -4896,24 +4892,44 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Map 1 <- Map 4 (BROADCAST_EDGE) + Map 2 <- Map 1 (BROADCAST_EDGE) + Reducer 3 <- Map 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 - Map 3 + Map 2 Map Operator Tree: TableScan alias: srcpart_hour - filterExpr: (hr is not null and (hr = 13)) (type: boolean) + filterExpr: (hr = 13) (type: boolean) Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (hr is not null and (hr = 13)) (type: boolean) - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: '13' (type: string) - sort order: + - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + predicate: (hr = 13) (type: boolean) + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 '13' (type: string) + 1 '13' (type: string) + input vertices: + 0 Map 1 + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) Map 4 Map Operator Tree: TableScan @@ -4928,7 +4944,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: ds (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reducer 2 + Reducer 3 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) diff --git a/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out b/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out index ef11095..ed94ee8 100644 --- a/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out +++ b/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out @@ -584,10 +584,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: d1 - filterExpr: (id is not null and (id = 1)) (type: boolean) + filterExpr: (id = 1) (type: boolean) Statistics: Num rows: 3 Data size: 15 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (id is not null and (id = 1)) (type: boolean) + predicate: (id = 1) (type: boolean) Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: 1 (type: int) @@ -597,35 +597,32 @@ STAGE PLANS: Map Operator Tree: TableScan alias: agg - filterExpr: (dim_shops_id is not null and (dim_shops_id = 1)) (type: boolean) + filterExpr: (dim_shops_id = 1) (type: boolean) Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 condition expressions: - 0 {amount} {dim_shops_id} + 0 {amount} 1 keys: - 0 dim_shops_id (type: int) + 0 1 (type: int) 1 1 (type: int) - outputColumnNames: _col0, _col1 + outputColumnNames: _col0 input vertices: 1 Map 1 Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((_col1 = 1) and (_col1 = 1)) (type: boolean) - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Select Operator - expressions: _col0 (type: decimal(10,0)) - outputColumnNames: _col0 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: _col0 (type: decimal(10,0)) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/tez/filter_join_breaktask.q.out b/ql/src/test/results/clientpositive/tez/filter_join_breaktask.q.out index b94ffe2..e48cf7c 100644 --- a/ql/src/test/results/clientpositive/tez/filter_join_breaktask.q.out +++ b/ql/src/test/results/clientpositive/tez/filter_join_breaktask.q.out @@ -217,13 +217,13 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: ((value <> '') and value is not null) (type: boolean) - Statistics: Num rows: 13 Data size: 109 Basic stats: COMPLETE Column stats: NONE + predicate: (value <> '') (type: boolean) + Statistics: Num rows: 25 Data size: 211 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: value (type: string) sort order: + Map-reduce partition columns: value (type: string) - Statistics: Num rows: 13 Data size: 109 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 211 Basic stats: COMPLETE Column stats: NONE tag: 1 auto parallelism: true Path -> Alias: @@ -375,17 +375,17 @@ STAGE PLANS: 1 {KEY.reducesinkkey0} outputColumnNames: _col0, _col13 Position of Big Table: 0 - Statistics: Num rows: 15 Data size: 130 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 232 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col13 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 15 Data size: 130 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 232 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 15 Data size: 130 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 232 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git a/ql/src/test/results/clientpositive/tez/mapjoin_mapjoin.q.out b/ql/src/test/results/clientpositive/tez/mapjoin_mapjoin.q.out index 2a3e56c..b261f37 100644 --- a/ql/src/test/results/clientpositive/tez/mapjoin_mapjoin.q.out +++ b/ql/src/test/results/clientpositive/tez/mapjoin_mapjoin.q.out @@ -539,13 +539,13 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (value is not null and (value > 'val_450')) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (value > 'val_450') (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: value (type: string) sort order: + Map-reduce partition columns: value (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/tez/optimize_nullscan.q.out b/ql/src/test/results/clientpositive/tez/optimize_nullscan.q.out index 2af6134..c45f0db 100644 --- a/ql/src/test/results/clientpositive/tez/optimize_nullscan.q.out +++ b/ql/src/test/results/clientpositive/tez/optimize_nullscan.q.out @@ -1540,75 +1540,48 @@ STAGE PLANS: #### A masked pattern was here #### Vertices: Map 1 + Map 4 Map Operator Tree: TableScan - alias: srcpart - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + alias: src GatherStats: false - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - tag: 1 - auto parallelism: true - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: hr=11 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - hr 11 - properties: - COLUMN_STATS_ACCURATE true - bucket_count -1 - columns key,value - columns.comments defaultdefault - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - numFiles 1 - numRows 500 - partition_columns ds/hr - partition_columns.types string:string - rawDataSize 5312 - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 + Filter Operator + isSamplingPred: false + predicate: false (type: boolean) + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns key,value - columns.comments defaultdefault - columns.types string:string + NumFilesPerFileSink: 1 #### A masked pattern was here #### - name default.srcpart - partition_columns ds/hr - partition_columns.types string:string - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0 + columns.types string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Path -> Alias: #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcpart - name: default.srcpart + Path -> Partition: #### A masked pattern was here #### Partition - base file name: hr=12 + base file name: src input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - hr 12 properties: COLUMN_STATS_ACCURATE true bucket_count -1 @@ -1616,13 +1589,11 @@ STAGE PLANS: columns.comments defaultdefault columns.types string:string #### A masked pattern was here #### - name default.srcpart + name default.src numFiles 1 numRows 500 - partition_columns ds/hr - partition_columns.types string:string rawDataSize 5312 - serialization.ddl struct srcpart { string key, string value} + serialization.ddl struct src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 5812 @@ -1632,29 +1603,51 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: + COLUMN_STATS_ACCURATE true bucket_count -1 columns key,value columns.comments defaultdefault columns.types string:string #### A masked pattern was here #### - name default.srcpart - partition_columns ds/hr - partition_columns.types string:string - serialization.ddl struct srcpart { string key, string value} + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcpart - name: default.srcpart -#### A masked pattern was here #### + name: default.src + name: default.src + Truncated Path -> Alias: + /src [src] + Map 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: false (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + tag: 0 + auto parallelism: true + Path -> Alias: + -mr-10002default.src{} [src] + Path -> Partition: + -mr-10002default.src{} Partition - base file name: hr=11 - input format: org.apache.hadoop.mapred.TextInputFormat + base file name: src + input format: org.apache.hadoop.hive.ql.io.OneNullRowInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-09 - hr 11 properties: COLUMN_STATS_ACCURATE true bucket_count -1 @@ -1662,129 +1655,172 @@ STAGE PLANS: columns.comments defaultdefault columns.types string:string #### A masked pattern was here #### - name default.srcpart + name default.src numFiles 1 numRows 500 - partition_columns ds/hr - partition_columns.types string:string rawDataSize 5312 - serialization.ddl struct srcpart { string key, string value} + serialization.ddl struct src { string key, string value} serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serialization.lib org.apache.hadoop.hive.serde2.NullStructSerDe totalSize 5812 #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.NullStructSerDe input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: + COLUMN_STATS_ACCURATE true bucket_count -1 columns key,value columns.comments defaultdefault columns.types string:string #### A masked pattern was here #### - name default.srcpart - partition_columns ds/hr - partition_columns.types string:string - serialization.ddl struct srcpart { string key, string value} + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcpart - name: default.srcpart + name: default.src + name: default.src + Truncated Path -> Alias: + -mr-10002default.src{} [src] + Reducer 2 + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + condition expressions: + 0 {KEY.reducesinkkey0} + 1 + outputColumnNames: _col0 + Position of Big Table: 0 + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 #### A masked pattern was here #### - Partition - base file name: hr=12 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-09 - hr 12 - properties: - COLUMN_STATS_ACCURATE true - bucket_count -1 - columns key,value - columns.comments defaultdefault - columns.types string:string + NumFilesPerFileSink: 1 #### A masked pattern was here #### - name default.srcpart - numFiles 1 - numRows 500 - partition_columns ds/hr - partition_columns.types string:string - rawDataSize 5312 - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0 + columns.types string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Union 3 + Vertex: Union 3 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * from (select key from src union all select src.key from src left outer join srcpart on src.key = srcpart.key) a where false +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@srcpart #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns key,value - columns.comments defaultdefault - columns.types string:string +POSTHOOK: query: select * from (select key from src union all select src.key from src left outer join srcpart on src.key = srcpart.key) a where false +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@srcpart #### A masked pattern was here #### - name default.srcpart - partition_columns ds/hr - partition_columns.types string:string - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +PREHOOK: query: explain extended +select * from src s1, src s2 where false and s1.value = s2.value +PREHOOK: type: QUERY +POSTHOOK: query: explain extended +select * from src s1, src s2 where false and s1.value = s2.value +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_JOIN + TOK_TABREF + TOK_TABNAME + src + s1 + TOK_TABREF + TOK_TABNAME + src + s2 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_WHERE + and + false + = + . + TOK_TABLE_OR_COL + s1 + value + . + TOK_TABLE_OR_COL + s2 + value + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcpart - name: default.srcpart - Truncated Path -> Alias: - /srcpart/ds=2008-04-08/hr=11 [srcpart] - /srcpart/ds=2008-04-08/hr=12 [srcpart] - /srcpart/ds=2008-04-09/hr=11 [srcpart] - /srcpart/ds=2008-04-09/hr=12 [srcpart] - Map 4 + Vertices: + Map 1 Map Operator Tree: TableScan - alias: src + alias: s2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: false (type: boolean) - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - columns _col0 - columns.types string - escape.delim \ - hive.serialization.extend.nesting.levels true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + tag: 1 + value expressions: key (type: string) + auto parallelism: true Path -> Alias: -#### A masked pattern was here #### + -mr-10003default.src{} [s2] Path -> Partition: -#### A masked pattern was here #### + -mr-10003default.src{} Partition base file name: src - input format: org.apache.hadoop.mapred.TextInputFormat + input format: org.apache.hadoop.hive.ql.io.OneNullRowInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: COLUMN_STATS_ACCURATE true @@ -1799,10 +1835,10 @@ STAGE PLANS: rawDataSize 5312 serialization.ddl struct src { string key, string value} serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serialization.lib org.apache.hadoop.hive.serde2.NullStructSerDe totalSize 5812 #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.NullStructSerDe input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1826,27 +1862,32 @@ STAGE PLANS: name: default.src name: default.src Truncated Path -> Alias: - /src [src] - Map 5 + -mr-10003default.src{} [s2] + Map 3 Map Operator Tree: TableScan - alias: src + alias: s1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - tag: 0 - auto parallelism: true + Filter Operator + isSamplingPred: false + predicate: false (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + tag: 0 + value expressions: key (type: string) + auto parallelism: true Path -> Alias: -#### A masked pattern was here #### + -mr-10002default.src{} [s1] Path -> Partition: -#### A masked pattern was here #### + -mr-10002default.src{} Partition base file name: src - input format: org.apache.hadoop.mapred.TextInputFormat + input format: org.apache.hadoop.hive.ql.io.OneNullRowInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: COLUMN_STATS_ACCURATE true @@ -1861,10 +1902,10 @@ STAGE PLANS: rawDataSize 5312 serialization.ddl struct src { string key, string value} serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serialization.lib org.apache.hadoop.hive.serde2.NullStructSerDe totalSize 5812 #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.NullStructSerDe input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1888,49 +1929,44 @@ STAGE PLANS: name: default.src name: default.src Truncated Path -> Alias: - /src [src] + -mr-10002default.src{} [s1] Reducer 2 Needs Tagging: false Reduce Operator Tree: Merge Join Operator condition map: - Left Outer Join0 to 1 + Inner Join 0 to 1 condition expressions: - 0 {KEY.reducesinkkey0} - 1 - outputColumnNames: _col0 + 0 {VALUE._col0} {KEY.reducesinkkey0} + 1 {VALUE._col0} {KEY.reducesinkkey0} + outputColumnNames: _col0, _col1, _col5, _col6 Position of Big Table: 0 - Filter Operator - isSamplingPred: false - predicate: false (type: boolean) - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - columns _col0 - columns.types string - escape.delim \ - hive.serialization.extend.nesting.levels true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Union 3 - Vertex: Union 3 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types string:string:string:string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -1938,21 +1974,11 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: select * from (select key from src union all select src.key from src left outer join srcpart on src.key = srcpart.key) a where false +PREHOOK: query: select * from src s1, src s2 where false and s1.value = s2.value PREHOOK: type: QUERY PREHOOK: Input: default@src -PREHOOK: Input: default@srcpart -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### -POSTHOOK: query: select * from (select key from src union all select src.key from src left outer join srcpart on src.key = srcpart.key) a where false +POSTHOOK: query: select * from src s1, src s2 where false and s1.value = s2.value POSTHOOK: type: QUERY POSTHOOK: Input: default@src -POSTHOOK: Input: default@srcpart -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/tez/sample1.q.out b/ql/src/test/results/clientpositive/tez/sample1.q.out index 48d861b..5a6237c 100644 --- a/ql/src/test/results/clientpositive/tez/sample1.q.out +++ b/ql/src/test/results/clientpositive/tez/sample1.q.out @@ -79,7 +79,7 @@ STAGE PLANS: predicate: (((hash(rand()) & 2147483647) % 1) = 0) (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), ds (type: string), hr (type: string) + expressions: UDFToInteger(key) (type: int), value (type: string), '2008-04-08' (type: string), '11' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/tez/skewjoin.q.out b/ql/src/test/results/clientpositive/tez/skewjoin.q.out new file mode 100644 index 0000000..e42326b --- /dev/null +++ b/ql/src/test/results/clientpositive/tez/skewjoin.q.out @@ -0,0 +1,1161 @@ +PREHOOK: query: CREATE TABLE T1(key STRING, val STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@T1 +POSTHOOK: query: CREATE TABLE T1(key STRING, val STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@T1 +PREHOOK: query: CREATE TABLE T2(key STRING, val STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@T2 +POSTHOOK: query: CREATE TABLE T2(key STRING, val STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@T2 +PREHOOK: query: CREATE TABLE T3(key STRING, val STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@T3 +POSTHOOK: query: CREATE TABLE T3(key STRING, val STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@T3 +PREHOOK: query: CREATE TABLE T4(key STRING, val STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@T4 +POSTHOOK: query: CREATE TABLE T4(key STRING, val STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@T4 +PREHOOK: query: CREATE TABLE dest_j1(key INT, value STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dest_j1 +POSTHOOK: query: CREATE TABLE dest_j1(key INT, value STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dest_j1 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/T1.txt' INTO TABLE T1 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@t1 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/T1.txt' INTO TABLE T1 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@t1 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/T2.txt' INTO TABLE T2 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@t2 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/T2.txt' INTO TABLE T2 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@t2 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/T3.txt' INTO TABLE T3 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@t3 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/T3.txt' INTO TABLE T3 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@t3 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/T1.txt' INTO TABLE T4 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@t4 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/T1.txt' INTO TABLE T4 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@t4 +PREHOOK: query: EXPLAIN +FROM src src1 JOIN src src2 ON (src1.key = src2.key) +INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +FROM src src1 JOIN src src2 ON (src1.key = src2.key) +INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Map 3 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {KEY.reducesinkkey0} + 1 {VALUE._col0} + outputColumnNames: _col0, _col6 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + + Stage: Stage-3 + Stats-Aggr Operator + +PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) +INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dest_j1 +POSTHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) +INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dest_j1 +POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT sum(hash(key)), sum(hash(value)) FROM dest_j1 +PREHOOK: type: QUERY +PREHOOK: Input: default@dest_j1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(key)), sum(hash(value)) FROM dest_j1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest_j1 +#### A masked pattern was here #### +278697 101852390308 +PREHOOK: query: EXPLAIN +SELECT /*+ STREAMTABLE(a) */ * +FROM T1 a JOIN T2 b ON a.key = b.key + JOIN T3 c ON b.key = c.key + JOIN T4 d ON c.key = d.key +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT /*+ STREAMTABLE(a) */ * +FROM T1 a JOIN T2 b ON a.key = b.key + JOIN T3 c ON b.key = c.key + JOIN T4 d ON c.key = d.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: d + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val (type: string) + Map 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val (type: string) + Map 4 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val (type: string) + Map 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val (type: string) + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + Inner Join 2 to 3 + condition expressions: + 0 {KEY.reducesinkkey0} {VALUE._col0} + 1 {KEY.reducesinkkey0} {VALUE._col0} + 2 {KEY.reducesinkkey0} {VALUE._col0} + 3 {KEY.reducesinkkey0} {VALUE._col0} + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11, _col15, _col16 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string), _col15 (type: string), _col16 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT /*+ STREAMTABLE(a) */ * +FROM T1 a JOIN T2 b ON a.key = b.key + JOIN T3 c ON b.key = c.key + JOIN T4 d ON c.key = d.key +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 +PREHOOK: Input: default@t3 +PREHOOK: Input: default@t4 +#### A masked pattern was here #### +POSTHOOK: query: SELECT /*+ STREAMTABLE(a) */ * +FROM T1 a JOIN T2 b ON a.key = b.key + JOIN T3 c ON b.key = c.key + JOIN T4 d ON c.key = d.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t2 +POSTHOOK: Input: default@t3 +POSTHOOK: Input: default@t4 +#### A masked pattern was here #### +2 12 2 22 2 12 2 12 +PREHOOK: query: EXPLAIN +SELECT /*+ STREAMTABLE(a,c) */ * +FROM T1 a JOIN T2 b ON a.key = b.key + JOIN T3 c ON b.key = c.key + JOIN T4 d ON c.key = d.key +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT /*+ STREAMTABLE(a,c) */ * +FROM T1 a JOIN T2 b ON a.key = b.key + JOIN T3 c ON b.key = c.key + JOIN T4 d ON c.key = d.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: d + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val (type: string) + Map 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val (type: string) + Map 4 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val (type: string) + Map 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: val (type: string) + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + Inner Join 2 to 3 + condition expressions: + 0 {KEY.reducesinkkey0} {VALUE._col0} + 1 {KEY.reducesinkkey0} {VALUE._col0} + 2 {KEY.reducesinkkey0} {VALUE._col0} + 3 {KEY.reducesinkkey0} {VALUE._col0} + outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11, _col15, _col16 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string), _col15 (type: string), _col16 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT /*+ STREAMTABLE(a,c) */ * +FROM T1 a JOIN T2 b ON a.key = b.key + JOIN T3 c ON b.key = c.key + JOIN T4 d ON c.key = d.key +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 +PREHOOK: Input: default@t3 +PREHOOK: Input: default@t4 +#### A masked pattern was here #### +POSTHOOK: query: SELECT /*+ STREAMTABLE(a,c) */ * +FROM T1 a JOIN T2 b ON a.key = b.key + JOIN T3 c ON b.key = c.key + JOIN T4 d ON c.key = d.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t2 +POSTHOOK: Input: default@t3 +POSTHOOK: Input: default@t4 +#### A masked pattern was here #### +2 12 2 22 2 12 2 12 +PREHOOK: query: EXPLAIN FROM T1 a JOIN src c ON c.key+1=a.key SELECT /*+ STREAMTABLE(a) */ sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN FROM T1 a JOIN src c ON c.key+1=a.key SELECT /*+ STREAMTABLE(a) */ sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key + 1) is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (key + 1) (type: double) + sort order: + + Map-reduce partition columns: (key + 1) (type: double) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: string) + Map 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: UDFToDouble(key) is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(key) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(key) (type: double) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string), val (type: string) + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {VALUE._col0} {VALUE._col1} + 1 {VALUE._col0} + outputColumnNames: _col0, _col1, _col5 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col5 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(hash(_col0)), sum(hash(_col1)), sum(hash(_col5)) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint) + Reducer 3 + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: FROM T1 a JOIN src c ON c.key+1=a.key SELECT /*+ STREAMTABLE(a) */ sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@t1 +#### A masked pattern was here #### +POSTHOOK: query: FROM T1 a JOIN src c ON c.key+1=a.key SELECT /*+ STREAMTABLE(a) */ sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@t1 +#### A masked pattern was here #### +198 6274 194 +PREHOOK: query: EXPLAIN FROM +(SELECT src.* FROM src) x +JOIN +(SELECT src.* FROM src) Y +ON (x.key = Y.key) +SELECT sum(hash(Y.key)), sum(hash(Y.value)) +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN FROM +(SELECT src.* FROM src) x +JOIN +(SELECT src.* FROM src) Y +ON (x.key = Y.key) +SELECT sum(hash(Y.key)), sum(hash(Y.value)) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {KEY.reducesinkkey0} {VALUE._col0} + outputColumnNames: _col2, _col3 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col2 (type: string), _col3 (type: string) + outputColumnNames: _col2, _col3 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(hash(_col2)), sum(hash(_col3)) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Reducer 3 + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0), sum(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: FROM +(SELECT src.* FROM src) x +JOIN +(SELECT src.* FROM src) Y +ON (x.key = Y.key) +SELECT sum(hash(Y.key)), sum(hash(Y.value)) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: FROM +(SELECT src.* FROM src) x +JOIN +(SELECT src.* FROM src) Y +ON (x.key = Y.key) +SELECT sum(hash(Y.key)), sum(hash(Y.value)) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +44481300 101852390308 +PREHOOK: query: EXPLAIN FROM +(SELECT src.* FROM src) x +JOIN +(SELECT src.* FROM src) Y +ON (x.key = Y.key and substring(x.value, 5)=substring(y.value, 5)+1) +SELECT sum(hash(Y.key)), sum(hash(Y.value)) +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN FROM +(SELECT src.* FROM src) x +JOIN +(SELECT src.* FROM src) Y +ON (x.key = Y.key and substring(x.value, 5)=substring(y.value, 5)+1) +SELECT sum(hash(Y.key)), sum(hash(Y.value)) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and UDFToDouble(substring(value, 5)) is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), UDFToDouble(substring(_col1, 5)) (type: double) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), UDFToDouble(substring(_col1, 5)) (type: double) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and (substring(value, 5) + 1) is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), (substring(_col1, 5) + 1) (type: double) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), (substring(_col1, 5) + 1) (type: double) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {KEY.reducesinkkey0} {VALUE._col0} + outputColumnNames: _col2, _col3 + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col2 (type: string), _col3 (type: string) + outputColumnNames: _col2, _col3 + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(hash(_col2)), sum(hash(_col3)) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Reducer 3 + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0), sum(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: FROM +(SELECT src.* FROM src) x +JOIN +(SELECT src.* FROM src) Y +ON (x.key = Y.key and substring(x.value, 5)=substring(y.value, 5)+1) +SELECT sum(hash(Y.key)), sum(hash(Y.value)) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: FROM +(SELECT src.* FROM src) x +JOIN +(SELECT src.* FROM src) Y +ON (x.key = Y.key and substring(x.value, 5)=substring(y.value, 5)+1) +SELECT sum(hash(Y.key)), sum(hash(Y.value)) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +NULL NULL +PREHOOK: query: EXPLAIN +SELECT sum(hash(src1.c1)), sum(hash(src2.c4)) +FROM +(SELECT src.key as c1, src.value as c2 from src) src1 +JOIN +(SELECT src.key as c3, src.value as c4 from src) src2 +ON src1.c1 = src2.c3 AND src1.c1 < 100 +JOIN +(SELECT src.key as c5, src.value as c6 from src) src3 +ON src1.c1 = src3.c5 AND src3.c5 < 80 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT sum(hash(src1.c1)), sum(hash(src2.c4)) +FROM +(SELECT src.key as c1, src.value as c2 from src) src1 +JOIN +(SELECT src.key as c3, src.value as c4 from src) src2 +ON src1.c1 = src2.c3 AND src1.c1 < 100 +JOIN +(SELECT src.key as c5, src.value as c6 from src) src3 +ON src1.c1 = src3.c5 AND src3.c5 < 80 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((key < 100) and (key < 80)) and key is not null) (type: boolean) + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Map 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key < 80) and (key < 100)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + Map 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key < 100) and (key < 80)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + condition expressions: + 0 {KEY.reducesinkkey0} + 1 {VALUE._col0} + 2 + outputColumnNames: _col0, _col3 + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col3 (type: string) + outputColumnNames: _col0, _col3 + Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(hash(_col0)), sum(hash(_col3)) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Reducer 3 + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0), sum(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT sum(hash(src1.c1)), sum(hash(src2.c4)) +FROM +(SELECT src.key as c1, src.value as c2 from src) src1 +JOIN +(SELECT src.key as c3, src.value as c4 from src) src2 +ON src1.c1 = src2.c3 AND src1.c1 < 100 +JOIN +(SELECT src.key as c5, src.value as c6 from src) src3 +ON src1.c1 = src3.c5 AND src3.c5 < 80 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(src1.c1)), sum(hash(src2.c4)) +FROM +(SELECT src.key as c1, src.value as c2 from src) src1 +JOIN +(SELECT src.key as c3, src.value as c4 from src) src2 +ON src1.c1 = src2.c3 AND src1.c1 < 100 +JOIN +(SELECT src.key as c5, src.value as c6 from src) src3 +ON src1.c1 = src3.c5 AND src3.c5 < 80 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +293143 -136853010385 +PREHOOK: query: EXPLAIN +SELECT /*+ mapjoin(v)*/ sum(hash(k.key)), sum(hash(v.val)) FROM T1 k LEFT OUTER JOIN T1 v ON k.key+1=v.key +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT /*+ mapjoin(v)*/ sum(hash(k.key)), sum(hash(v.val)) FROM T1 k LEFT OUTER JOIN T1 v ON k.key+1=v.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: v + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(key) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(key) (type: double) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: val (type: string) + Map 4 + Map Operator Tree: + TableScan + alias: k + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: (key + 1) (type: double) + sort order: + + Map-reduce partition columns: (key + 1) (type: double) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: key (type: string) + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + condition expressions: + 0 {VALUE._col0} + 1 {VALUE._col1} + outputColumnNames: _col0, _col6 + Statistics: Num rows: 0 Data size: 33 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col6 + Statistics: Num rows: 0 Data size: 33 Basic stats: PARTIAL Column stats: NONE + Group By Operator + aggregations: sum(hash(_col0)), sum(hash(_col6)) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint) + Reducer 3 + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0), sum(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT /*+ mapjoin(v)*/ sum(hash(k.key)), sum(hash(v.val)) FROM T1 k LEFT OUTER JOIN T1 v ON k.key+1=v.key +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT /*+ mapjoin(v)*/ sum(hash(k.key)), sum(hash(v.val)) FROM T1 k LEFT OUTER JOIN T1 v ON k.key+1=v.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +#### A masked pattern was here #### +372 6320 +PREHOOK: query: select /*+ mapjoin(k)*/ sum(hash(k.key)), sum(hash(v.val)) from T1 k join T1 v on k.key=v.val +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +#### A masked pattern was here #### +POSTHOOK: query: select /*+ mapjoin(k)*/ sum(hash(k.key)), sum(hash(v.val)) from T1 k join T1 v on k.key=v.val +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +#### A masked pattern was here #### +NULL NULL +PREHOOK: query: select /*+ mapjoin(k)*/ sum(hash(k.key)), sum(hash(v.val)) from T1 k join T1 v on k.key=v.key +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +#### A masked pattern was here #### +POSTHOOK: query: select /*+ mapjoin(k)*/ sum(hash(k.key)), sum(hash(v.val)) from T1 k join T1 v on k.key=v.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +#### A masked pattern was here #### +429 12643 +PREHOOK: query: select sum(hash(k.key)), sum(hash(v.val)) from T1 k join T1 v on k.key=v.key +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +#### A masked pattern was here #### +POSTHOOK: query: select sum(hash(k.key)), sum(hash(v.val)) from T1 k join T1 v on k.key=v.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +#### A masked pattern was here #### +429 12643 +PREHOOK: query: select count(1) from T1 a join T1 b on a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +#### A masked pattern was here #### +POSTHOOK: query: select count(1) from T1 a join T1 b on a.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +#### A masked pattern was here #### +8 +PREHOOK: query: FROM T1 a LEFT OUTER JOIN T2 c ON c.key+1=a.key SELECT sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 +#### A masked pattern was here #### +POSTHOOK: query: FROM T1 a LEFT OUTER JOIN T2 c ON c.key+1=a.key SELECT sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t2 +#### A masked pattern was here #### +317 9462 50 +PREHOOK: query: FROM T1 a RIGHT OUTER JOIN T2 c ON c.key+1=a.key SELECT /*+ STREAMTABLE(a) */ sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 +#### A masked pattern was here #### +POSTHOOK: query: FROM T1 a RIGHT OUTER JOIN T2 c ON c.key+1=a.key SELECT /*+ STREAMTABLE(a) */ sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t2 +#### A masked pattern was here #### +51 1570 318 +PREHOOK: query: FROM T1 a FULL OUTER JOIN T2 c ON c.key+1=a.key SELECT /*+ STREAMTABLE(a) */ sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 +#### A masked pattern was here #### +POSTHOOK: query: FROM T1 a FULL OUTER JOIN T2 c ON c.key+1=a.key SELECT /*+ STREAMTABLE(a) */ sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t2 +#### A masked pattern was here #### +317 9462 318 +PREHOOK: query: SELECT sum(hash(src1.key)), sum(hash(src1.val)), sum(hash(src2.key)) FROM T1 src1 LEFT OUTER JOIN T2 src2 ON src1.key+1 = src2.key RIGHT OUTER JOIN T2 src3 ON src2.key = src3.key +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(src1.key)), sum(hash(src1.val)), sum(hash(src2.key)) FROM T1 src1 LEFT OUTER JOIN T2 src2 ON src1.key+1 = src2.key RIGHT OUTER JOIN T2 src3 ON src2.key = src3.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t2 +#### A masked pattern was here #### +370 11003 377 +PREHOOK: query: SELECT sum(hash(src1.key)), sum(hash(src1.val)), sum(hash(src2.key)) FROM T1 src1 JOIN T2 src2 ON src1.key+1 = src2.key JOIN T2 src3 ON src2.key = src3.key +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(src1.key)), sum(hash(src1.val)), sum(hash(src2.key)) FROM T1 src1 JOIN T2 src2 ON src1.key+1 = src2.key JOIN T2 src3 ON src2.key = src3.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t2 +#### A masked pattern was here #### +370 11003 377 +PREHOOK: query: select /*+ mapjoin(v)*/ sum(hash(k.key)), sum(hash(v.val)) from T1 k left outer join T1 v on k.key+1=v.key +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +#### A masked pattern was here #### +POSTHOOK: query: select /*+ mapjoin(v)*/ sum(hash(k.key)), sum(hash(v.val)) from T1 k left outer join T1 v on k.key+1=v.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +#### A masked pattern was here #### +372 6320 diff --git a/ql/src/test/results/clientpositive/tez/subquery_exists.q.out b/ql/src/test/results/clientpositive/tez/subquery_exists.q.out index c79b718..24f2d10 100644 --- a/ql/src/test/results/clientpositive/tez/subquery_exists.q.out +++ b/ql/src/test/results/clientpositive/tez/subquery_exists.q.out @@ -48,22 +48,22 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((value > 'val_9') and value is not null) and key is not null) (type: boolean) - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + predicate: ((value > 'val_9') and key is not null) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: string), key (type: string) outputColumnNames: _col1, _col2 - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col1 (type: string), _col2 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Merge Join Operator diff --git a/ql/src/test/results/clientpositive/tez/subquery_in.q.out b/ql/src/test/results/clientpositive/tez/subquery_in.q.out index d983c11..b35a266 100644 --- a/ql/src/test/results/clientpositive/tez/subquery_in.q.out +++ b/ql/src/test/results/clientpositive/tez/subquery_in.q.out @@ -127,22 +127,22 @@ STAGE PLANS: alias: s1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > '9') and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key > '9') (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Map 3 Map Operator Tree: TableScan @@ -258,22 +258,22 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key > '9') and key is not null) and value is not null) (type: boolean) - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > '9') and value is not null) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Merge Join Operator @@ -696,22 +696,22 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key > '9') and key is not null) and value is not null) (type: boolean) - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > '9') and value is not null) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: key, value - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: key (type: string), value (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Merge Join Operator @@ -739,21 +739,21 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 21 Data size: 223 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 21 Data size: 223 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 21 Data size: 223 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 21 Data size: 223 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/tez/tez_smb_1.q.out b/ql/src/test/results/clientpositive/tez/tez_smb_1.q.out index 0bb2ec8..e65ef01 100644 --- a/ql/src/test/results/clientpositive/tez/tez_smb_1.q.out +++ b/ql/src/test/results/clientpositive/tez/tez_smb_1.q.out @@ -186,504 +186,3 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: select s1.key, s1.value, s3.value from tab s1 join tab s3 on s1.key=s3.key -PREHOOK: type: QUERY -PREHOOK: Input: default@tab -PREHOOK: Input: default@tab@ds=2008-04-08 -#### A masked pattern was here #### -POSTHOOK: query: select s1.key, s1.value, s3.value from tab s1 join tab s3 on s1.key=s3.key -POSTHOOK: type: QUERY -POSTHOOK: Input: default@tab -POSTHOOK: Input: default@tab@ds=2008-04-08 -#### A masked pattern was here #### -0 val_0 val_0 -0 val_0 val_0 -0 val_0 val_0 -0 val_0 val_0 -0 val_0 val_0 -0 val_0 val_0 -0 val_0 val_0 -0 val_0 val_0 -0 val_0 val_0 -2 val_2 val_2 -4 val_4 val_4 -8 val_8 val_8 -20 val_20 val_20 -24 val_24 val_24 -24 val_24 val_24 -24 val_24 val_24 -24 val_24 val_24 -26 val_26 val_26 -26 val_26 val_26 -26 val_26 val_26 -26 val_26 val_26 -28 val_28 val_28 -42 val_42 val_42 -42 val_42 val_42 -42 val_42 val_42 -42 val_42 val_42 -44 val_44 val_44 -64 val_64 val_64 -66 val_66 val_66 -80 val_80 val_80 -82 val_82 val_82 -84 val_84 val_84 -84 val_84 val_84 -84 val_84 val_84 -84 val_84 val_84 -86 val_86 val_86 -114 val_114 val_114 -116 val_116 val_116 -118 val_118 val_118 -118 val_118 val_118 -118 val_118 val_118 -118 val_118 val_118 -134 val_134 val_134 -134 val_134 val_134 -134 val_134 val_134 -134 val_134 val_134 -136 val_136 val_136 -138 val_138 val_138 -138 val_138 val_138 -138 val_138 val_138 -138 val_138 val_138 -138 val_138 val_138 -138 val_138 val_138 -138 val_138 val_138 -138 val_138 val_138 -138 val_138 val_138 -138 val_138 val_138 -138 val_138 val_138 -138 val_138 val_138 -138 val_138 val_138 -138 val_138 val_138 -138 val_138 val_138 -138 val_138 val_138 -150 val_150 val_150 -152 val_152 val_152 -152 val_152 val_152 -152 val_152 val_152 -152 val_152 val_152 -156 val_156 val_156 -158 val_158 val_158 -170 val_170 val_170 -172 val_172 val_172 -172 val_172 val_172 -172 val_172 val_172 -172 val_172 val_172 -174 val_174 val_174 -174 val_174 val_174 -174 val_174 val_174 -174 val_174 val_174 -176 val_176 val_176 -176 val_176 val_176 -176 val_176 val_176 -176 val_176 val_176 -178 val_178 val_178 -190 val_190 val_190 -192 val_192 val_192 -194 val_194 val_194 -196 val_196 val_196 -200 val_200 val_200 -200 val_200 val_200 -200 val_200 val_200 -200 val_200 val_200 -202 val_202 val_202 -208 val_208 val_208 -208 val_208 val_208 -208 val_208 val_208 -208 val_208 val_208 -208 val_208 val_208 -208 val_208 val_208 -208 val_208 val_208 -208 val_208 val_208 -208 val_208 val_208 -222 val_222 val_222 -224 val_224 val_224 -224 val_224 val_224 -224 val_224 val_224 -224 val_224 val_224 -226 val_226 val_226 -228 val_228 val_228 -242 val_242 val_242 -242 val_242 val_242 -242 val_242 val_242 -242 val_242 val_242 -244 val_244 val_244 -248 val_248 val_248 -260 val_260 val_260 -262 val_262 val_262 -266 val_266 val_266 -280 val_280 val_280 -280 val_280 val_280 -280 val_280 val_280 -280 val_280 val_280 -282 val_282 val_282 -282 val_282 val_282 -282 val_282 val_282 -282 val_282 val_282 -284 val_284 val_284 -286 val_286 val_286 -288 val_288 val_288 -288 val_288 val_288 -288 val_288 val_288 -288 val_288 val_288 -310 val_310 val_310 -316 val_316 val_316 -316 val_316 val_316 -316 val_316 val_316 -316 val_316 val_316 -316 val_316 val_316 -316 val_316 val_316 -316 val_316 val_316 -316 val_316 val_316 -316 val_316 val_316 -318 val_318 val_318 -318 val_318 val_318 -318 val_318 val_318 -318 val_318 val_318 -318 val_318 val_318 -318 val_318 val_318 -318 val_318 val_318 -318 val_318 val_318 -318 val_318 val_318 -332 val_332 val_332 -336 val_336 val_336 -338 val_338 val_338 -356 val_356 val_356 -374 val_374 val_374 -378 val_378 val_378 -392 val_392 val_392 -394 val_394 val_394 -396 val_396 val_396 -396 val_396 val_396 -396 val_396 val_396 -396 val_396 val_396 -396 val_396 val_396 -396 val_396 val_396 -396 val_396 val_396 -396 val_396 val_396 -396 val_396 val_396 -400 val_400 val_400 -402 val_402 val_402 -404 val_404 val_404 -404 val_404 val_404 -404 val_404 val_404 -404 val_404 val_404 -406 val_406 val_406 -406 val_406 val_406 -406 val_406 val_406 -406 val_406 val_406 -406 val_406 val_406 -406 val_406 val_406 -406 val_406 val_406 -406 val_406 val_406 -406 val_406 val_406 -406 val_406 val_406 -406 val_406 val_406 -406 val_406 val_406 -406 val_406 val_406 -406 val_406 val_406 -406 val_406 val_406 -406 val_406 val_406 -424 val_424 val_424 -424 val_424 val_424 -424 val_424 val_424 -424 val_424 val_424 -444 val_444 val_444 -446 val_446 val_446 -448 val_448 val_448 -460 val_460 val_460 -462 val_462 val_462 -462 val_462 val_462 -462 val_462 val_462 -462 val_462 val_462 -466 val_466 val_466 -466 val_466 val_466 -466 val_466 val_466 -466 val_466 val_466 -466 val_466 val_466 -466 val_466 val_466 -466 val_466 val_466 -466 val_466 val_466 -466 val_466 val_466 -468 val_468 val_468 -468 val_468 val_468 -468 val_468 val_468 -468 val_468 val_468 -468 val_468 val_468 -468 val_468 val_468 -468 val_468 val_468 -468 val_468 val_468 -468 val_468 val_468 -468 val_468 val_468 -468 val_468 val_468 -468 val_468 val_468 -468 val_468 val_468 -468 val_468 val_468 -468 val_468 val_468 -468 val_468 val_468 -480 val_480 val_480 -480 val_480 val_480 -480 val_480 val_480 -480 val_480 val_480 -480 val_480 val_480 -480 val_480 val_480 -480 val_480 val_480 -480 val_480 val_480 -480 val_480 val_480 -482 val_482 val_482 -484 val_484 val_484 -11 val_11 val_11 -15 val_15 val_15 -15 val_15 val_15 -15 val_15 val_15 -15 val_15 val_15 -17 val_17 val_17 -19 val_19 val_19 -33 val_33 val_33 -35 val_35 val_35 -35 val_35 val_35 -35 val_35 val_35 -35 val_35 val_35 -35 val_35 val_35 -35 val_35 val_35 -35 val_35 val_35 -35 val_35 val_35 -35 val_35 val_35 -37 val_37 val_37 -37 val_37 val_37 -37 val_37 val_37 -37 val_37 val_37 -51 val_51 val_51 -51 val_51 val_51 -51 val_51 val_51 -51 val_51 val_51 -53 val_53 val_53 -57 val_57 val_57 -77 val_77 val_77 -95 val_95 val_95 -95 val_95 val_95 -95 val_95 val_95 -95 val_95 val_95 -97 val_97 val_97 -97 val_97 val_97 -97 val_97 val_97 -97 val_97 val_97 -103 val_103 val_103 -103 val_103 val_103 -103 val_103 val_103 -103 val_103 val_103 -105 val_105 val_105 -125 val_125 val_125 -125 val_125 val_125 -125 val_125 val_125 -125 val_125 val_125 -129 val_129 val_129 -129 val_129 val_129 -129 val_129 val_129 -129 val_129 val_129 -143 val_143 val_143 -145 val_145 val_145 -149 val_149 val_149 -149 val_149 val_149 -149 val_149 val_149 -149 val_149 val_149 -163 val_163 val_163 -165 val_165 val_165 -165 val_165 val_165 -165 val_165 val_165 -165 val_165 val_165 -167 val_167 val_167 -167 val_167 val_167 -167 val_167 val_167 -167 val_167 val_167 -167 val_167 val_167 -167 val_167 val_167 -167 val_167 val_167 -167 val_167 val_167 -167 val_167 val_167 -169 val_169 val_169 -169 val_169 val_169 -169 val_169 val_169 -169 val_169 val_169 -169 val_169 val_169 -169 val_169 val_169 -169 val_169 val_169 -169 val_169 val_169 -169 val_169 val_169 -169 val_169 val_169 -169 val_169 val_169 -169 val_169 val_169 -169 val_169 val_169 -169 val_169 val_169 -169 val_169 val_169 -169 val_169 val_169 -181 val_181 val_181 -183 val_183 val_183 -187 val_187 val_187 -187 val_187 val_187 -187 val_187 val_187 -187 val_187 val_187 -187 val_187 val_187 -187 val_187 val_187 -187 val_187 val_187 -187 val_187 val_187 -187 val_187 val_187 -189 val_189 val_189 -213 val_213 val_213 -213 val_213 val_213 -213 val_213 val_213 -213 val_213 val_213 -217 val_217 val_217 -217 val_217 val_217 -217 val_217 val_217 -217 val_217 val_217 -219 val_219 val_219 -219 val_219 val_219 -219 val_219 val_219 -219 val_219 val_219 -233 val_233 val_233 -233 val_233 val_233 -233 val_233 val_233 -233 val_233 val_233 -235 val_235 val_235 -237 val_237 val_237 -237 val_237 val_237 -237 val_237 val_237 -237 val_237 val_237 -239 val_239 val_239 -239 val_239 val_239 -239 val_239 val_239 -239 val_239 val_239 -255 val_255 val_255 -255 val_255 val_255 -255 val_255 val_255 -255 val_255 val_255 -257 val_257 val_257 -273 val_273 val_273 -273 val_273 val_273 -273 val_273 val_273 -273 val_273 val_273 -273 val_273 val_273 -273 val_273 val_273 -273 val_273 val_273 -273 val_273 val_273 -273 val_273 val_273 -275 val_275 val_275 -277 val_277 val_277 -277 val_277 val_277 -277 val_277 val_277 -277 val_277 val_277 -277 val_277 val_277 -277 val_277 val_277 -277 val_277 val_277 -277 val_277 val_277 -277 val_277 val_277 -277 val_277 val_277 -277 val_277 val_277 -277 val_277 val_277 -277 val_277 val_277 -277 val_277 val_277 -277 val_277 val_277 -277 val_277 val_277 -291 val_291 val_291 -305 val_305 val_305 -307 val_307 val_307 -307 val_307 val_307 -307 val_307 val_307 -307 val_307 val_307 -309 val_309 val_309 -309 val_309 val_309 -309 val_309 val_309 -309 val_309 val_309 -321 val_321 val_321 -321 val_321 val_321 -321 val_321 val_321 -321 val_321 val_321 -323 val_323 val_323 -325 val_325 val_325 -325 val_325 val_325 -325 val_325 val_325 -325 val_325 val_325 -327 val_327 val_327 -327 val_327 val_327 -327 val_327 val_327 -327 val_327 val_327 -327 val_327 val_327 -327 val_327 val_327 -327 val_327 val_327 -327 val_327 val_327 -327 val_327 val_327 -341 val_341 val_341 -345 val_345 val_345 -365 val_365 val_365 -367 val_367 val_367 -367 val_367 val_367 -367 val_367 val_367 -367 val_367 val_367 -369 val_369 val_369 -369 val_369 val_369 -369 val_369 val_369 -369 val_369 val_369 -369 val_369 val_369 -369 val_369 val_369 -369 val_369 val_369 -369 val_369 val_369 -369 val_369 val_369 -389 val_389 val_389 -411 val_411 val_411 -413 val_413 val_413 -413 val_413 val_413 -413 val_413 val_413 -413 val_413 val_413 -417 val_417 val_417 -417 val_417 val_417 -417 val_417 val_417 -417 val_417 val_417 -417 val_417 val_417 -417 val_417 val_417 -417 val_417 val_417 -417 val_417 val_417 -417 val_417 val_417 -419 val_419 val_419 -431 val_431 val_431 -431 val_431 val_431 -431 val_431 val_431 -431 val_431 val_431 -431 val_431 val_431 -431 val_431 val_431 -431 val_431 val_431 -431 val_431 val_431 -431 val_431 val_431 -435 val_435 val_435 -437 val_437 val_437 -439 val_439 val_439 -439 val_439 val_439 -439 val_439 val_439 -439 val_439 val_439 -453 val_453 val_453 -455 val_455 val_455 -457 val_457 val_457 -459 val_459 val_459 -459 val_459 val_459 -459 val_459 val_459 -459 val_459 val_459 -475 val_475 val_475 -477 val_477 val_477 -479 val_479 val_479 -491 val_491 val_491 -493 val_493 val_493 -495 val_495 val_495 -497 val_497 val_497 -PREHOOK: query: select count(*) from tab s2 -PREHOOK: type: QUERY -PREHOOK: Input: default@tab -PREHOOK: Input: default@tab@ds=2008-04-08 -#### A masked pattern was here #### -POSTHOOK: query: select count(*) from tab s2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@tab -POSTHOOK: Input: default@tab@ds=2008-04-08 -#### A masked pattern was here #### -242 diff --git a/ql/src/test/results/clientpositive/tez/transform_ppr2.q.out b/ql/src/test/results/clientpositive/tez/transform_ppr2.q.out index abf627f..9bdb5f4 100644 --- a/ql/src/test/results/clientpositive/tez/transform_ppr2.q.out +++ b/ql/src/test/results/clientpositive/tez/transform_ppr2.q.out @@ -111,7 +111,7 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE GatherStats: false Select Operator - expressions: ds (type: string), key (type: string), value (type: string) + expressions: '2008-04-08' (type: string), key (type: string), value (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Transform Operator diff --git a/ql/src/test/results/clientpositive/tez/union7.q.out b/ql/src/test/results/clientpositive/tez/union7.q.out index 2971029..91dc888 100644 --- a/ql/src/test/results/clientpositive/tez/union7.q.out +++ b/ql/src/test/results/clientpositive/tez/union7.q.out @@ -92,14 +92,14 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 600 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 600 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 600 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/tez/update_all_partitioned.q.out b/ql/src/test/results/clientpositive/tez/update_all_partitioned.q.out index 533dd88..286e2ed 100644 --- a/ql/src/test/results/clientpositive/tez/update_all_partitioned.q.out +++ b/ql/src/test/results/clientpositive/tez/update_all_partitioned.q.out @@ -85,21 +85,15 @@ POSTHOOK: Input: default@acid_uap@ds=today POSTHOOK: Input: default@acid_uap@ds=tomorrow #### A masked pattern was here #### -1073279343 fred today --1073279343 oj1YrV5Wa today -1073051226 fred today -1072910839 fred today --1072081801 dPkN74F7 today -1072081801 fred today -1072076362 fred today -1071480828 fred today --1071363017 Anj0oF today -1071363017 fred today --1070883071 0ruyd6Y50JpdGRf6HqD today -1070883071 fred today -1070551679 fred today --1070551679 iUR3Q today -1069736047 fred today --1069736047 k17Am8uPHWk02cEf1jet today 762 fred tomorrow 762 fred tomorrow 762 fred tomorrow diff --git a/ql/src/test/results/clientpositive/tez/vector_bucket.q.out b/ql/src/test/results/clientpositive/tez/vector_bucket.q.out new file mode 100644 index 0000000..e4deb04 --- /dev/null +++ b/ql/src/test/results/clientpositive/tez/vector_bucket.q.out @@ -0,0 +1,105 @@ +PREHOOK: query: CREATE TABLE non_orc_table(a INT, b STRING) CLUSTERED BY(a) INTO 2 BUCKETS STORED AS sequencefile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@non_orc_table +POSTHOOK: query: CREATE TABLE non_orc_table(a INT, b STRING) CLUSTERED BY(a) INTO 2 BUCKETS STORED AS sequencefile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@non_orc_table +PREHOOK: query: explain +insert into table non_orc_table values(1, 'one'),(1, 'one'), (2, 'two'),(3, 'three') +PREHOOK: type: QUERY +POSTHOOK: query: explain +insert into table non_orc_table values(1, 'one'),(1, 'one'), (2, 'two'),(3, 'three') +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: values__tmp__table__1 + Statistics: Num rows: 0 Data size: 26 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: tmp_values_col1 (type: string), tmp_values_col2 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 26 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + sort order: + Map-reduce partition columns: UDFToInteger(_col0) (type: int) + Statistics: Num rows: 0 Data size: 26 Basic stats: PARTIAL Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + Reducer 2 + Reduce Operator Tree: + Extract + Statistics: Num rows: 0 Data size: 26 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 26 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 26 Basic stats: PARTIAL Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.non_orc_table + Execution mode: vectorized + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.non_orc_table + + Stage: Stage-3 + Stats-Aggr Operator + +PREHOOK: query: select a, b from non_orc_table order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@non_orc_table +#### A masked pattern was here #### +POSTHOOK: query: select a, b from non_orc_table order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@non_orc_table +#### A masked pattern was here #### +PREHOOK: query: insert into table non_orc_table values(1, 'one'),(1, 'one'), (2, 'two'),(3, 'three') +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__2 +PREHOOK: Output: default@non_orc_table +POSTHOOK: query: insert into table non_orc_table values(1, 'one'),(1, 'one'), (2, 'two'),(3, 'three') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__2 +POSTHOOK: Output: default@non_orc_table +POSTHOOK: Lineage: non_orc_table.a EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: non_orc_table.b SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: select a, b from non_orc_table order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@non_orc_table +#### A masked pattern was here #### +POSTHOOK: query: select a, b from non_orc_table order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@non_orc_table +#### A masked pattern was here #### +1 one +1 one +2 two +3 three diff --git a/ql/src/test/results/clientpositive/tez/vector_char_2.q.out b/ql/src/test/results/clientpositive/tez/vector_char_2.q.out index 2e66485..59f872a 100644 --- a/ql/src/test/results/clientpositive/tez/vector_char_2.q.out +++ b/ql/src/test/results/clientpositive/tez/vector_char_2.q.out @@ -110,6 +110,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 250 Data size: 49500 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) + Execution mode: vectorized Reducer 3 Reduce Operator Tree: Select Operator @@ -241,6 +242,7 @@ STAGE PLANS: sort order: - Statistics: Num rows: 250 Data size: 49500 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) + Execution mode: vectorized Reducer 3 Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/tez/vector_data_types.q.out b/ql/src/test/results/clientpositive/tez/vector_data_types.q.out index a3bf59d..3f38a45 100644 --- a/ql/src/test/results/clientpositive/tez/vector_data_types.q.out +++ b/ql/src/test/results/clientpositive/tez/vector_data_types.q.out @@ -177,6 +177,17 @@ NULL 409 65536 4294967490 46.97 25.92 false fred miller 2013-03-01 09:11:58.7031 120 331 65539 4294967324 88.02 40.94 true holly nixon 2013-03-01 09:11:58.703262 96.64 yard duty 71 298 65540 4294967510 91.63 9.6 false gabriella zipper 2013-03-01 09:11:58.70314 68.14 geology 114 385 65541 4294967458 73.48 34.97 true oscar quirinius 2013-03-01 09:11:58.703143 72.33 xylophone band +PREHOOK: query: SELECT SUM(HASH(*)) +FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i) as q +PREHOOK: type: QUERY +PREHOOK: Input: default@over1korc +#### A masked pattern was here #### +POSTHOOK: query: SELECT SUM(HASH(*)) +FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i) as q +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1korc +#### A masked pattern was here #### +-17045922556 PREHOOK: query: EXPLAIN select t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i LIMIT 20 PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN select t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i LIMIT 20 @@ -239,23 +250,34 @@ POSTHOOK: query: SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc OR POSTHOOK: type: QUERY POSTHOOK: Input: default@over1korc #### A masked pattern was here #### -108 301 65536 4294967357 90.05 17.59 true ethan johnson 1860-11-12 20:05:55.011470936 75.70 undecided -118 497 65536 4294967381 50.32 12.72 false david nixon 1860-11-12 20:05:55.011484936 83.48 values clariffication -18 280 65536 4294967320 32.92 45.94 false holly white 1860-11-12 20:05:55.011285936 58.86 topologyariffication -69 489 65536 4294967404 33.52 17.99 false oscar ichabod 1860-11-12 20:05:55.011446936 32.68 topologyariffication -27 405 65536 4294967508 82.24 29.41 true oscar ovid 1860-11-12 20:05:55.011365936 16.85 biologyariffication -42 495 65536 4294967431 43.57 46.81 false tom johnson 1860-11-12 20:05:55.011444936 62.25 american historytion -NULL 409 65536 4294967490 46.97 25.92 false fred miller 1860-11-12 20:05:55.011315936 33.45 history historytion -59 431 65537 4294967326 11.34 2.9 true oscar robinson 1860-11-12 20:05:55.011360936 81.04 zync studiesorytion -63 458 65537 4294967463 21.94 49.71 true fred van buren 1860-11-12 20:05:55.011477936 99.34 wind surfingorytion -121 355 65537 4294967437 85.9 10.99 true calvin brown 1860-11-12 20:05:55.011453936 4.49 yard dutyngorytion -72 267 65537 4294967460 55.2 42.89 false oscar carson 1860-11-12 20:05:55.011352936 51.91 topologyngorytion -22 264 65537 4294967419 8.07 10.71 false david xylophone 1860-11-12 20:05:55.011335936 11.87 undecidedngorytion -101 283 65538 4294967527 84.03 27.29 true yuri brown 1860-11-12 20:05:55.011505936 44.24 biologydngorytion -109 376 65538 4294967453 59.61 35.62 true priscilla xylophone 1860-11-12 20:05:55.011485936 9.53 study skillsorytion -83 353 65538 4294967339 0.08 23.91 true holly thompson 1860-11-12 20:05:55.011356936 30.27 quiet hoursorytion -29 331 65539 4294967420 73.18 28.96 true ethan brown 1860-11-12 20:05:55.011293936 58.85 zync studiesorytion -70 430 65539 4294967534 18.89 43.84 true tom carson 1860-11-12 20:05:55.011381936 21.93 joggyingiesorytion -120 331 65539 4294967324 88.02 40.94 true holly nixon 1860-11-12 20:05:55.011461936 96.64 yard dutyesorytion -71 298 65540 4294967510 91.63 9.6 false gabriella zipper 1860-11-12 20:05:55.011339936 68.14 geologyyesorytion -114 385 65541 4294967458 73.48 34.97 true oscar quirinius 1860-11-12 20:05:55.011342936 72.33 xylophone bandytion +108 301 65536 4294967357 90.05 17.59 true ethan johnson 2013-03-01 09:11:58.703271 75.70 undecided +118 497 65536 4294967381 50.32 12.72 false david nixon 2013-03-01 09:11:58.703285 83.48 values clariffication +18 280 65536 4294967320 32.92 45.94 false holly white 2013-03-01 09:11:58.703086 58.86 topology +69 489 65536 4294967404 33.52 17.99 false oscar ichabod 2013-03-01 09:11:58.703247 32.68 topology +27 405 65536 4294967508 82.24 29.41 true oscar ovid 2013-03-01 09:11:58.703166 16.85 biology +42 495 65536 4294967431 43.57 46.81 false tom johnson 2013-03-01 09:11:58.703245 62.25 american history +NULL 409 65536 4294967490 46.97 25.92 false fred miller 2013-03-01 09:11:58.703116 33.45 history +59 431 65537 4294967326 11.34 2.9 true oscar robinson 2013-03-01 09:11:58.703161 81.04 zync studies +63 458 65537 4294967463 21.94 49.71 true fred van buren 2013-03-01 09:11:58.703278 99.34 wind surfing +121 355 65537 4294967437 85.9 10.99 true calvin brown 2013-03-01 09:11:58.703254 4.49 yard duty +72 267 65537 4294967460 55.2 42.89 false oscar carson 2013-03-01 09:11:58.703153 51.91 topology +22 264 65537 4294967419 8.07 10.71 false david xylophone 2013-03-01 09:11:58.703136 11.87 undecided +101 283 65538 4294967527 84.03 27.29 true yuri brown 2013-03-01 09:11:58.703306 44.24 biology +109 376 65538 4294967453 59.61 35.62 true priscilla xylophone 2013-03-01 09:11:58.703286 9.53 study skills +83 353 65538 4294967339 0.08 23.91 true holly thompson 2013-03-01 09:11:58.703157 30.27 quiet hour +29 331 65539 4294967420 73.18 28.96 true ethan brown 2013-03-01 09:11:58.703094 58.85 zync studies +70 430 65539 4294967534 18.89 43.84 true tom carson 2013-03-01 09:11:58.703182 21.93 joggying +120 331 65539 4294967324 88.02 40.94 true holly nixon 2013-03-01 09:11:58.703262 96.64 yard duty +71 298 65540 4294967510 91.63 9.6 false gabriella zipper 2013-03-01 09:11:58.70314 68.14 geology +114 385 65541 4294967458 73.48 34.97 true oscar quirinius 2013-03-01 09:11:58.703143 72.33 xylophone band +PREHOOK: query: SELECT SUM(HASH(*)) +FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i) as q +PREHOOK: type: QUERY +PREHOOK: Input: default@over1korc +#### A masked pattern was here #### +POSTHOOK: query: SELECT SUM(HASH(*)) +FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i) as q +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1korc +#### A masked pattern was here #### +-17045922556 diff --git a/ql/src/test/results/clientpositive/tez/vector_groupby_reduce.q.out b/ql/src/test/results/clientpositive/tez/vector_groupby_reduce.q.out new file mode 100644 index 0000000..d45fbde --- /dev/null +++ b/ql/src/test/results/clientpositive/tez/vector_groupby_reduce.q.out @@ -0,0 +1,466 @@ +PREHOOK: query: create table store_sales_txt +( + ss_sold_date_sk int, + ss_sold_time_sk int, + ss_item_sk int, + ss_customer_sk int, + ss_cdemo_sk int, + ss_hdemo_sk int, + ss_addr_sk int, + ss_store_sk int, + ss_promo_sk int, + ss_ticket_number int, + ss_quantity int, + ss_wholesale_cost float, + ss_list_price float, + ss_sales_price float, + ss_ext_discount_amt float, + ss_ext_sales_price float, + ss_ext_wholesale_cost float, + ss_ext_list_price float, + ss_ext_tax float, + ss_coupon_amt float, + ss_net_paid float, + ss_net_paid_inc_tax float, + ss_net_profit float +) +row format delimited fields terminated by '|' +stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@store_sales_txt +POSTHOOK: query: create table store_sales_txt +( + ss_sold_date_sk int, + ss_sold_time_sk int, + ss_item_sk int, + ss_customer_sk int, + ss_cdemo_sk int, + ss_hdemo_sk int, + ss_addr_sk int, + ss_store_sk int, + ss_promo_sk int, + ss_ticket_number int, + ss_quantity int, + ss_wholesale_cost float, + ss_list_price float, + ss_sales_price float, + ss_ext_discount_amt float, + ss_ext_sales_price float, + ss_ext_wholesale_cost float, + ss_ext_list_price float, + ss_ext_tax float, + ss_coupon_amt float, + ss_net_paid float, + ss_net_paid_inc_tax float, + ss_net_profit float +) +row format delimited fields terminated by '|' +stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@store_sales_txt +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/store_sales.txt' OVERWRITE INTO TABLE store_sales_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@store_sales_txt +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/store_sales.txt' OVERWRITE INTO TABLE store_sales_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@store_sales_txt +PREHOOK: query: create table store_sales +( + ss_sold_date_sk int, + ss_sold_time_sk int, + ss_item_sk int, + ss_customer_sk int, + ss_cdemo_sk int, + ss_hdemo_sk int, + ss_addr_sk int, + ss_store_sk int, + ss_promo_sk int, + ss_ticket_number int, + ss_quantity int, + ss_wholesale_cost float, + ss_list_price float, + ss_sales_price float, + ss_ext_discount_amt float, + ss_ext_sales_price float, + ss_ext_wholesale_cost float, + ss_ext_list_price float, + ss_ext_tax float, + ss_coupon_amt float, + ss_net_paid float, + ss_net_paid_inc_tax float, + ss_net_profit float +) +stored as orc +tblproperties ("orc.stripe.size"="33554432", "orc.compress.size"="16384") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@store_sales +POSTHOOK: query: create table store_sales +( + ss_sold_date_sk int, + ss_sold_time_sk int, + ss_item_sk int, + ss_customer_sk int, + ss_cdemo_sk int, + ss_hdemo_sk int, + ss_addr_sk int, + ss_store_sk int, + ss_promo_sk int, + ss_ticket_number int, + ss_quantity int, + ss_wholesale_cost float, + ss_list_price float, + ss_sales_price float, + ss_ext_discount_amt float, + ss_ext_sales_price float, + ss_ext_wholesale_cost float, + ss_ext_list_price float, + ss_ext_tax float, + ss_coupon_amt float, + ss_net_paid float, + ss_net_paid_inc_tax float, + ss_net_profit float +) +stored as orc +tblproperties ("orc.stripe.size"="33554432", "orc.compress.size"="16384") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@store_sales +PREHOOK: query: insert overwrite table store_sales +select +ss_sold_date_sk , + ss_sold_time_sk , + ss_item_sk , + ss_customer_sk , + ss_cdemo_sk , + ss_hdemo_sk , + ss_addr_sk , + ss_store_sk , + ss_promo_sk , + ss_ticket_number , + ss_quantity , + ss_wholesale_cost , + ss_list_price , + ss_sales_price , + ss_ext_discount_amt , + ss_ext_sales_price , + ss_ext_wholesale_cost , + ss_ext_list_price , + ss_ext_tax , + ss_coupon_amt , + ss_net_paid , + ss_net_paid_inc_tax , + ss_net_profit + from store_sales_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@store_sales_txt +PREHOOK: Output: default@store_sales +POSTHOOK: query: insert overwrite table store_sales +select +ss_sold_date_sk , + ss_sold_time_sk , + ss_item_sk , + ss_customer_sk , + ss_cdemo_sk , + ss_hdemo_sk , + ss_addr_sk , + ss_store_sk , + ss_promo_sk , + ss_ticket_number , + ss_quantity , + ss_wholesale_cost , + ss_list_price , + ss_sales_price , + ss_ext_discount_amt , + ss_ext_sales_price , + ss_ext_wholesale_cost , + ss_ext_list_price , + ss_ext_tax , + ss_coupon_amt , + ss_net_paid , + ss_net_paid_inc_tax , + ss_net_profit + from store_sales_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@store_sales_txt +POSTHOOK: Output: default@store_sales +POSTHOOK: Lineage: store_sales.ss_addr_sk SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_addr_sk, type:int, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_cdemo_sk SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_cdemo_sk, type:int, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_coupon_amt SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_coupon_amt, type:float, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_customer_sk SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_customer_sk, type:int, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_ext_discount_amt SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_ext_discount_amt, type:float, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_ext_list_price SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_ext_list_price, type:float, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_ext_sales_price SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_ext_sales_price, type:float, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_ext_tax SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_ext_tax, type:float, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_ext_wholesale_cost SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_ext_wholesale_cost, type:float, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_hdemo_sk SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_hdemo_sk, type:int, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_item_sk SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_item_sk, type:int, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_list_price SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_list_price, type:float, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_net_paid SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_net_paid, type:float, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_net_paid_inc_tax SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_net_paid_inc_tax, type:float, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_net_profit SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_net_profit, type:float, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_promo_sk SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_promo_sk, type:int, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_quantity SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_quantity, type:int, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_sales_price SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_sales_price, type:float, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_sold_date_sk SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_sold_date_sk, type:int, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_sold_time_sk SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_sold_time_sk, type:int, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_store_sk SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_store_sk, type:int, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_ticket_number SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_ticket_number, type:int, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_wholesale_cost SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_wholesale_cost, type:float, comment:null), ] +PREHOOK: query: explain +select + ss_ticket_number +from + store_sales +group by ss_ticket_number +limit 20 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select + ss_ticket_number +from + store_sales +group by ss_ticket_number +limit 20 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: store_sales + Statistics: Num rows: 1000 Data size: 88276 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ss_ticket_number (type: int) + outputColumnNames: ss_ticket_number + Statistics: Num rows: 1000 Data size: 88276 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: ss_ticket_number (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1000 Data size: 88276 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1000 Data size: 88276 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized + Reducer 2 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 44138 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 44138 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 20 + Statistics: Num rows: 20 Data size: 1760 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 20 Data size: 1760 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: 20 + Processor Tree: + ListSink + +PREHOOK: query: select + ss_ticket_number +from + store_sales +group by ss_ticket_number +limit 20 +PREHOOK: type: QUERY +PREHOOK: Input: default@store_sales +#### A masked pattern was here #### +POSTHOOK: query: select + ss_ticket_number +from + store_sales +group by ss_ticket_number +limit 20 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@store_sales +#### A masked pattern was here #### +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +PREHOOK: query: explain +select + min(ss_ticket_number) +from + (select + ss_ticket_number + from + store_sales + group by ss_ticket_number) a +group by ss_ticket_number +limit 20 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select + min(ss_ticket_number) +from + (select + ss_ticket_number + from + store_sales + group by ss_ticket_number) a +group by ss_ticket_number +limit 20 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: store_sales + Statistics: Num rows: 1000 Data size: 88276 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ss_ticket_number (type: int) + outputColumnNames: ss_ticket_number + Statistics: Num rows: 1000 Data size: 88276 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: ss_ticket_number (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1000 Data size: 88276 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1000 Data size: 88276 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized + Reducer 2 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 44138 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 44138 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: min(_col0) + keys: _col0 (type: int) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 22069 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 22069 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 20 + Statistics: Num rows: 20 Data size: 1760 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 20 Data size: 1760 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: 20 + Processor Tree: + ListSink + +PREHOOK: query: select + min(ss_ticket_number) +from + (select + ss_ticket_number + from + store_sales + group by ss_ticket_number) a +group by ss_ticket_number +limit 20 +PREHOOK: type: QUERY +PREHOOK: Input: default@store_sales +#### A masked pattern was here #### +POSTHOOK: query: select + min(ss_ticket_number) +from + (select + ss_ticket_number + from + store_sales + group by ss_ticket_number) a +group by ss_ticket_number +limit 20 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@store_sales +#### A masked pattern was here #### +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 diff --git a/ql/src/test/results/clientpositive/tez/vector_partitioned_date_time.q.out b/ql/src/test/results/clientpositive/tez/vector_partitioned_date_time.q.out new file mode 100644 index 0000000..dce61d5 --- /dev/null +++ b/ql/src/test/results/clientpositive/tez/vector_partitioned_date_time.q.out @@ -0,0 +1,2053 @@ +PREHOOK: query: -- Check if vectorization code is handling partitioning on DATE and the other data types. + + +CREATE TABLE flights_tiny ( + origin_city_name STRING, + dest_city_name STRING, + fl_date DATE, + arr_delay FLOAT, + fl_num INT +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@flights_tiny +POSTHOOK: query: -- Check if vectorization code is handling partitioning on DATE and the other data types. + + +CREATE TABLE flights_tiny ( + origin_city_name STRING, + dest_city_name STRING, + fl_date DATE, + arr_delay FLOAT, + fl_num INT +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@flights_tiny +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/flights_tiny.txt.1' OVERWRITE INTO TABLE flights_tiny +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@flights_tiny +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/flights_tiny.txt.1' OVERWRITE INTO TABLE flights_tiny +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@flights_tiny +PREHOOK: query: CREATE TABLE flights_tiny_orc STORED AS ORC AS +SELECT origin_city_name, dest_city_name, fl_date, to_utc_timestamp(fl_date, 'America/Los_Angeles') as fl_time, arr_delay, fl_num +FROM flights_tiny +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@flights_tiny +PREHOOK: Output: database:default +PREHOOK: Output: default@flights_tiny_orc +POSTHOOK: query: CREATE TABLE flights_tiny_orc STORED AS ORC AS +SELECT origin_city_name, dest_city_name, fl_date, to_utc_timestamp(fl_date, 'America/Los_Angeles') as fl_time, arr_delay, fl_num +FROM flights_tiny +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@flights_tiny +POSTHOOK: Output: database:default +POSTHOOK: Output: default@flights_tiny_orc +PREHOOK: query: SELECT * FROM flights_tiny_orc +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM flights_tiny_orc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc +#### A masked pattern was here #### +Baltimore New York 2010-10-20 2010-10-20 07:00:00 -30.0 1064 +Baltimore New York 2010-10-20 2010-10-20 07:00:00 23.0 1142 +Baltimore New York 2010-10-20 2010-10-20 07:00:00 6.0 1599 +Chicago New York 2010-10-20 2010-10-20 07:00:00 42.0 361 +Chicago New York 2010-10-20 2010-10-20 07:00:00 24.0 897 +Chicago New York 2010-10-20 2010-10-20 07:00:00 15.0 1531 +Chicago New York 2010-10-20 2010-10-20 07:00:00 -6.0 1610 +Chicago New York 2010-10-20 2010-10-20 07:00:00 -2.0 3198 +Baltimore New York 2010-10-21 2010-10-21 07:00:00 17.0 1064 +Baltimore New York 2010-10-21 2010-10-21 07:00:00 105.0 1142 +Baltimore New York 2010-10-21 2010-10-21 07:00:00 28.0 1599 +Chicago New York 2010-10-21 2010-10-21 07:00:00 142.0 361 +Chicago New York 2010-10-21 2010-10-21 07:00:00 77.0 897 +Chicago New York 2010-10-21 2010-10-21 07:00:00 53.0 1531 +Chicago New York 2010-10-21 2010-10-21 07:00:00 -5.0 1610 +Chicago New York 2010-10-21 2010-10-21 07:00:00 51.0 3198 +Baltimore New York 2010-10-22 2010-10-22 07:00:00 -12.0 1064 +Baltimore New York 2010-10-22 2010-10-22 07:00:00 54.0 1142 +Baltimore New York 2010-10-22 2010-10-22 07:00:00 18.0 1599 +Chicago New York 2010-10-22 2010-10-22 07:00:00 2.0 361 +Chicago New York 2010-10-22 2010-10-22 07:00:00 24.0 897 +Chicago New York 2010-10-22 2010-10-22 07:00:00 16.0 1531 +Chicago New York 2010-10-22 2010-10-22 07:00:00 -6.0 1610 +Chicago New York 2010-10-22 2010-10-22 07:00:00 -11.0 3198 +Baltimore New York 2010-10-23 2010-10-23 07:00:00 18.0 272 +Baltimore New York 2010-10-23 2010-10-23 07:00:00 -10.0 1805 +Baltimore New York 2010-10-23 2010-10-23 07:00:00 6.0 3171 +Chicago New York 2010-10-23 2010-10-23 07:00:00 3.0 384 +Chicago New York 2010-10-23 2010-10-23 07:00:00 32.0 426 +Chicago New York 2010-10-23 2010-10-23 07:00:00 1.0 650 +Chicago New York 2010-10-23 2010-10-23 07:00:00 11.0 3085 +Baltimore New York 2010-10-24 2010-10-24 07:00:00 12.0 1599 +Baltimore New York 2010-10-24 2010-10-24 07:00:00 20.0 2571 +Chicago New York 2010-10-24 2010-10-24 07:00:00 10.0 361 +Chicago New York 2010-10-24 2010-10-24 07:00:00 113.0 897 +Chicago New York 2010-10-24 2010-10-24 07:00:00 -5.0 1531 +Chicago New York 2010-10-24 2010-10-24 07:00:00 -17.0 1610 +Chicago New York 2010-10-24 2010-10-24 07:00:00 -3.0 3198 +Baltimore New York 2010-10-25 2010-10-25 07:00:00 -25.0 1064 +Baltimore New York 2010-10-25 2010-10-25 07:00:00 92.0 1142 +Baltimore New York 2010-10-25 2010-10-25 07:00:00 106.0 1599 +Chicago New York 2010-10-25 2010-10-25 07:00:00 31.0 361 +Chicago New York 2010-10-25 2010-10-25 07:00:00 -1.0 897 +Chicago New York 2010-10-25 2010-10-25 07:00:00 43.0 1531 +Chicago New York 2010-10-25 2010-10-25 07:00:00 6.0 1610 +Chicago New York 2010-10-25 2010-10-25 07:00:00 -16.0 3198 +Baltimore New York 2010-10-26 2010-10-26 07:00:00 -22.0 1064 +Baltimore New York 2010-10-26 2010-10-26 07:00:00 123.0 1142 +Baltimore New York 2010-10-26 2010-10-26 07:00:00 90.0 1599 +Chicago New York 2010-10-26 2010-10-26 07:00:00 12.0 361 +Chicago New York 2010-10-26 2010-10-26 07:00:00 0.0 897 +Chicago New York 2010-10-26 2010-10-26 07:00:00 29.0 1531 +Chicago New York 2010-10-26 2010-10-26 07:00:00 -17.0 1610 +Chicago New York 2010-10-26 2010-10-26 07:00:00 6.0 3198 +Baltimore New York 2010-10-27 2010-10-27 07:00:00 -18.0 1064 +Baltimore New York 2010-10-27 2010-10-27 07:00:00 49.0 1142 +Baltimore New York 2010-10-27 2010-10-27 07:00:00 92.0 1599 +Chicago New York 2010-10-27 2010-10-27 07:00:00 148.0 361 +Chicago New York 2010-10-27 2010-10-27 07:00:00 -11.0 897 +Chicago New York 2010-10-27 2010-10-27 07:00:00 70.0 1531 +Chicago New York 2010-10-27 2010-10-27 07:00:00 8.0 1610 +Chicago New York 2010-10-27 2010-10-27 07:00:00 21.0 3198 +Baltimore New York 2010-10-28 2010-10-28 07:00:00 -4.0 1064 +Baltimore New York 2010-10-28 2010-10-28 07:00:00 -14.0 1142 +Baltimore New York 2010-10-28 2010-10-28 07:00:00 -14.0 1599 +Chicago New York 2010-10-28 2010-10-28 07:00:00 2.0 361 +Chicago New York 2010-10-28 2010-10-28 07:00:00 2.0 897 +Chicago New York 2010-10-28 2010-10-28 07:00:00 -11.0 1531 +Chicago New York 2010-10-28 2010-10-28 07:00:00 3.0 1610 +Chicago New York 2010-10-28 2010-10-28 07:00:00 -18.0 3198 +Baltimore New York 2010-10-29 2010-10-29 07:00:00 -24.0 1064 +Baltimore New York 2010-10-29 2010-10-29 07:00:00 21.0 1142 +Baltimore New York 2010-10-29 2010-10-29 07:00:00 -2.0 1599 +Chicago New York 2010-10-29 2010-10-29 07:00:00 -12.0 361 +Chicago New York 2010-10-29 2010-10-29 07:00:00 -11.0 897 +Chicago New York 2010-10-29 2010-10-29 07:00:00 15.0 1531 +Chicago New York 2010-10-29 2010-10-29 07:00:00 -18.0 1610 +Chicago New York 2010-10-29 2010-10-29 07:00:00 -4.0 3198 +Baltimore New York 2010-10-30 2010-10-30 07:00:00 14.0 272 +Baltimore New York 2010-10-30 2010-10-30 07:00:00 -1.0 1805 +Baltimore New York 2010-10-30 2010-10-30 07:00:00 5.0 3171 +Chicago New York 2010-10-30 2010-10-30 07:00:00 -6.0 384 +Chicago New York 2010-10-30 2010-10-30 07:00:00 -10.0 426 +Chicago New York 2010-10-30 2010-10-30 07:00:00 -5.0 650 +Chicago New York 2010-10-30 2010-10-30 07:00:00 -5.0 3085 +Baltimore New York 2010-10-31 2010-10-31 07:00:00 -1.0 1599 +Baltimore New York 2010-10-31 2010-10-31 07:00:00 -14.0 2571 +Chicago New York 2010-10-31 2010-10-31 07:00:00 -25.0 361 +Chicago New York 2010-10-31 2010-10-31 07:00:00 -18.0 897 +Chicago New York 2010-10-31 2010-10-31 07:00:00 -4.0 1531 +Chicago New York 2010-10-31 2010-10-31 07:00:00 -22.0 1610 +Chicago New York 2010-10-31 2010-10-31 07:00:00 -15.0 3198 +Cleveland New York 2010-10-30 2010-10-30 07:00:00 -23.0 2018 +Cleveland New York 2010-10-30 2010-10-30 07:00:00 -12.0 2932 +Cleveland New York 2010-10-29 2010-10-29 07:00:00 -4.0 2630 +Cleveland New York 2010-10-29 2010-10-29 07:00:00 -19.0 2646 +Cleveland New York 2010-10-29 2010-10-29 07:00:00 -12.0 3014 +Cleveland New York 2010-10-28 2010-10-28 07:00:00 3.0 2630 +Cleveland New York 2010-10-28 2010-10-28 07:00:00 -6.0 2646 +Cleveland New York 2010-10-28 2010-10-28 07:00:00 1.0 3014 +Cleveland New York 2010-10-27 2010-10-27 07:00:00 16.0 2630 +Cleveland New York 2010-10-27 2010-10-27 07:00:00 27.0 3014 +Cleveland New York 2010-10-26 2010-10-26 07:00:00 4.0 2630 +Cleveland New York 2010-10-26 2010-10-26 07:00:00 -27.0 2646 +Cleveland New York 2010-10-26 2010-10-26 07:00:00 -11.0 2662 +Cleveland New York 2010-10-26 2010-10-26 07:00:00 13.0 3014 +Cleveland New York 2010-10-25 2010-10-25 07:00:00 -4.0 2630 +Cleveland New York 2010-10-25 2010-10-25 07:00:00 81.0 2646 +Cleveland New York 2010-10-25 2010-10-25 07:00:00 42.0 3014 +Cleveland New York 2010-10-24 2010-10-24 07:00:00 5.0 2254 +Cleveland New York 2010-10-24 2010-10-24 07:00:00 -11.0 2630 +Cleveland New York 2010-10-24 2010-10-24 07:00:00 -20.0 2646 +Cleveland New York 2010-10-24 2010-10-24 07:00:00 -9.0 3014 +Cleveland New York 2010-10-23 2010-10-23 07:00:00 -21.0 2932 +Cleveland New York 2010-10-22 2010-10-22 07:00:00 1.0 2630 +Cleveland New York 2010-10-22 2010-10-22 07:00:00 -25.0 2646 +Cleveland New York 2010-10-22 2010-10-22 07:00:00 -3.0 3014 +Cleveland New York 2010-10-21 2010-10-21 07:00:00 3.0 2630 +Cleveland New York 2010-10-21 2010-10-21 07:00:00 29.0 2646 +Cleveland New York 2010-10-21 2010-10-21 07:00:00 72.0 3014 +Cleveland New York 2010-10-20 2010-10-20 07:00:00 -8.0 2630 +Cleveland New York 2010-10-20 2010-10-20 07:00:00 -15.0 3014 +Washington New York 2010-10-23 2010-10-23 07:00:00 -25.0 5832 +Washington New York 2010-10-23 2010-10-23 07:00:00 -21.0 5904 +Washington New York 2010-10-23 2010-10-23 07:00:00 -18.0 5917 +Washington New York 2010-10-30 2010-10-30 07:00:00 -27.0 5904 +Washington New York 2010-10-30 2010-10-30 07:00:00 -16.0 5917 +Washington New York 2010-10-20 2010-10-20 07:00:00 -2.0 7291 +Washington New York 2010-10-21 2010-10-21 07:00:00 22.0 7291 +Washington New York 2010-10-23 2010-10-23 07:00:00 -16.0 7274 +Washington New York 2010-10-24 2010-10-24 07:00:00 -26.0 7282 +Washington New York 2010-10-25 2010-10-25 07:00:00 9.0 7291 +Washington New York 2010-10-26 2010-10-26 07:00:00 4.0 7291 +Washington New York 2010-10-27 2010-10-27 07:00:00 26.0 7291 +Washington New York 2010-10-28 2010-10-28 07:00:00 45.0 7291 +Washington New York 2010-10-29 2010-10-29 07:00:00 1.0 7291 +Washington New York 2010-10-31 2010-10-31 07:00:00 -18.0 7282 +PREHOOK: query: select * from flights_tiny_orc sort by fl_num, fl_date limit 25 +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc +#### A masked pattern was here #### +POSTHOOK: query: select * from flights_tiny_orc sort by fl_num, fl_date limit 25 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc +#### A masked pattern was here #### +Baltimore New York 2010-10-23 2010-10-23 07:00:00 18.0 272 +Baltimore New York 2010-10-30 2010-10-30 07:00:00 14.0 272 +Chicago New York 2010-10-20 2010-10-20 07:00:00 42.0 361 +Chicago New York 2010-10-21 2010-10-21 07:00:00 142.0 361 +Chicago New York 2010-10-22 2010-10-22 07:00:00 2.0 361 +Chicago New York 2010-10-24 2010-10-24 07:00:00 10.0 361 +Chicago New York 2010-10-25 2010-10-25 07:00:00 31.0 361 +Chicago New York 2010-10-26 2010-10-26 07:00:00 12.0 361 +Chicago New York 2010-10-27 2010-10-27 07:00:00 148.0 361 +Chicago New York 2010-10-28 2010-10-28 07:00:00 2.0 361 +Chicago New York 2010-10-29 2010-10-29 07:00:00 -12.0 361 +Chicago New York 2010-10-31 2010-10-31 07:00:00 -25.0 361 +Chicago New York 2010-10-23 2010-10-23 07:00:00 3.0 384 +Chicago New York 2010-10-30 2010-10-30 07:00:00 -6.0 384 +Chicago New York 2010-10-23 2010-10-23 07:00:00 32.0 426 +Chicago New York 2010-10-30 2010-10-30 07:00:00 -10.0 426 +Chicago New York 2010-10-23 2010-10-23 07:00:00 1.0 650 +Chicago New York 2010-10-30 2010-10-30 07:00:00 -5.0 650 +Chicago New York 2010-10-20 2010-10-20 07:00:00 24.0 897 +Chicago New York 2010-10-21 2010-10-21 07:00:00 77.0 897 +Chicago New York 2010-10-22 2010-10-22 07:00:00 24.0 897 +Chicago New York 2010-10-24 2010-10-24 07:00:00 113.0 897 +Chicago New York 2010-10-25 2010-10-25 07:00:00 -1.0 897 +Chicago New York 2010-10-26 2010-10-26 07:00:00 0.0 897 +Chicago New York 2010-10-27 2010-10-27 07:00:00 -11.0 897 +PREHOOK: query: select fl_date, count(*) from flights_tiny_orc group by fl_date +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc +#### A masked pattern was here #### +POSTHOOK: query: select fl_date, count(*) from flights_tiny_orc group by fl_date +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc +#### A masked pattern was here #### +2010-10-20 11 +2010-10-21 12 +2010-10-22 11 +2010-10-23 12 +2010-10-24 12 +2010-10-25 12 +2010-10-26 13 +2010-10-27 11 +2010-10-28 12 +2010-10-29 12 +2010-10-30 11 +2010-10-31 8 +PREHOOK: query: explain +select * from flights_tiny_orc sort by fl_num, fl_date limit 25 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select * from flights_tiny_orc sort by fl_num, fl_date limit 25 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: flights_tiny_orc + Statistics: Num rows: 137 Data size: 39456 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), dest_city_name (type: string), fl_date (type: date), fl_time (type: timestamp), arr_delay (type: float), fl_num (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 137 Data size: 39456 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col5 (type: int), _col2 (type: date) + sort order: ++ + Statistics: Num rows: 137 Data size: 39456 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: timestamp), _col4 (type: float) + Execution mode: vectorized + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey1 (type: date), VALUE._col2 (type: timestamp), VALUE._col3 (type: float), KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 137 Data size: 39456 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 25 + Statistics: Num rows: 25 Data size: 7200 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col5 (type: int), _col2 (type: date) + sort order: ++ + Statistics: Num rows: 25 Data size: 7200 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: timestamp), _col4 (type: float) + Execution mode: vectorized + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey1 (type: date), VALUE._col2 (type: timestamp), VALUE._col3 (type: float), KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 25 Data size: 7200 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 25 + Statistics: Num rows: 25 Data size: 7200 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 25 Data size: 7200 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: 25 + Processor Tree: + ListSink + +PREHOOK: query: select * from flights_tiny_orc sort by fl_num, fl_date limit 25 +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc +#### A masked pattern was here #### +POSTHOOK: query: select * from flights_tiny_orc sort by fl_num, fl_date limit 25 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc +#### A masked pattern was here #### +Baltimore New York 2010-10-23 2010-10-23 07:00:00 18.0 272 +Baltimore New York 2010-10-30 2010-10-30 07:00:00 14.0 272 +Chicago New York 2010-10-20 2010-10-20 07:00:00 42.0 361 +Chicago New York 2010-10-21 2010-10-21 07:00:00 142.0 361 +Chicago New York 2010-10-22 2010-10-22 07:00:00 2.0 361 +Chicago New York 2010-10-24 2010-10-24 07:00:00 10.0 361 +Chicago New York 2010-10-25 2010-10-25 07:00:00 31.0 361 +Chicago New York 2010-10-26 2010-10-26 07:00:00 12.0 361 +Chicago New York 2010-10-27 2010-10-27 07:00:00 148.0 361 +Chicago New York 2010-10-28 2010-10-28 07:00:00 2.0 361 +Chicago New York 2010-10-29 2010-10-29 07:00:00 -12.0 361 +Chicago New York 2010-10-31 2010-10-31 07:00:00 -25.0 361 +Chicago New York 2010-10-23 2010-10-23 07:00:00 3.0 384 +Chicago New York 2010-10-30 2010-10-30 07:00:00 -6.0 384 +Chicago New York 2010-10-23 2010-10-23 07:00:00 32.0 426 +Chicago New York 2010-10-30 2010-10-30 07:00:00 -10.0 426 +Chicago New York 2010-10-23 2010-10-23 07:00:00 1.0 650 +Chicago New York 2010-10-30 2010-10-30 07:00:00 -5.0 650 +Chicago New York 2010-10-20 2010-10-20 07:00:00 24.0 897 +Chicago New York 2010-10-21 2010-10-21 07:00:00 77.0 897 +Chicago New York 2010-10-22 2010-10-22 07:00:00 24.0 897 +Chicago New York 2010-10-24 2010-10-24 07:00:00 113.0 897 +Chicago New York 2010-10-25 2010-10-25 07:00:00 -1.0 897 +Chicago New York 2010-10-26 2010-10-26 07:00:00 0.0 897 +Chicago New York 2010-10-27 2010-10-27 07:00:00 -11.0 897 +PREHOOK: query: explain +select fl_date, count(*) from flights_tiny_orc group by fl_date +PREHOOK: type: QUERY +POSTHOOK: query: explain +select fl_date, count(*) from flights_tiny_orc group by fl_date +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: flights_tiny_orc + Statistics: Num rows: 137 Data size: 39456 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: fl_date (type: date) + outputColumnNames: fl_date + Statistics: Num rows: 137 Data size: 39456 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: fl_date (type: date) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 137 Data size: 39456 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: date) + sort order: + + Map-reduce partition columns: _col0 (type: date) + Statistics: Num rows: 137 Data size: 39456 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: vectorized + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: date) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 68 Data size: 19584 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: date), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 68 Data size: 19584 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 68 Data size: 19584 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select fl_date, count(*) from flights_tiny_orc group by fl_date +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc +#### A masked pattern was here #### +POSTHOOK: query: select fl_date, count(*) from flights_tiny_orc group by fl_date +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc +#### A masked pattern was here #### +2010-10-20 11 +2010-10-21 12 +2010-10-22 11 +2010-10-23 12 +2010-10-24 12 +2010-10-25 12 +2010-10-26 13 +2010-10-27 11 +2010-10-28 12 +2010-10-29 12 +2010-10-30 11 +2010-10-31 8 +PREHOOK: query: CREATE TABLE flights_tiny_orc_partitioned_date ( + origin_city_name STRING, + dest_city_name STRING, + fl_time TIMESTAMP, + arr_delay FLOAT, + fl_num INT +) +PARTITIONED BY (fl_date DATE) +STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@flights_tiny_orc_partitioned_date +POSTHOOK: query: CREATE TABLE flights_tiny_orc_partitioned_date ( + origin_city_name STRING, + dest_city_name STRING, + fl_time TIMESTAMP, + arr_delay FLOAT, + fl_num INT +) +PARTITIONED BY (fl_date DATE) +STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@flights_tiny_orc_partitioned_date +PREHOOK: query: INSERT INTO TABLE flights_tiny_orc_partitioned_date +PARTITION (fl_date) +SELECT origin_city_name, dest_city_name, fl_time, arr_delay, fl_num, fl_date +FROM flights_tiny_orc +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc +PREHOOK: Output: default@flights_tiny_orc_partitioned_date +POSTHOOK: query: INSERT INTO TABLE flights_tiny_orc_partitioned_date +PARTITION (fl_date) +SELECT origin_city_name, dest_city_name, fl_time, arr_delay, fl_num, fl_date +FROM flights_tiny_orc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc +POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-20).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-20).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-20).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-20).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-20).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-21).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-21).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-21).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-21).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-21).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-22).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-22).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-22).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-22).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-22).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-23).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-23).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-23).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-23).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-23).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-24).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-24).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-24).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-24).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-24).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-25).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-25).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-25).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-25).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-25).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-26).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-26).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-26).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-26).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-26).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-27).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-27).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-27).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-27).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-27).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-28).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-28).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-28).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-28).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-28).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-29).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-29).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-29).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-29).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-29).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-30).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-30).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-30).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-30).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-30).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-31).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-31).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-31).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-31).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-31).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +PREHOOK: query: select * from flights_tiny_orc_partitioned_date +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc_partitioned_date +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 +#### A masked pattern was here #### +POSTHOOK: query: select * from flights_tiny_orc_partitioned_date +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 +#### A masked pattern was here #### +Baltimore New York 2010-10-20 07:00:00 -30.0 1064 2010-10-20 +Baltimore New York 2010-10-20 07:00:00 23.0 1142 2010-10-20 +Baltimore New York 2010-10-20 07:00:00 6.0 1599 2010-10-20 +Chicago New York 2010-10-20 07:00:00 42.0 361 2010-10-20 +Chicago New York 2010-10-20 07:00:00 24.0 897 2010-10-20 +Chicago New York 2010-10-20 07:00:00 15.0 1531 2010-10-20 +Chicago New York 2010-10-20 07:00:00 -6.0 1610 2010-10-20 +Chicago New York 2010-10-20 07:00:00 -2.0 3198 2010-10-20 +Cleveland New York 2010-10-20 07:00:00 -8.0 2630 2010-10-20 +Cleveland New York 2010-10-20 07:00:00 -15.0 3014 2010-10-20 +Washington New York 2010-10-20 07:00:00 -2.0 7291 2010-10-20 +Baltimore New York 2010-10-21 07:00:00 17.0 1064 2010-10-21 +Baltimore New York 2010-10-21 07:00:00 105.0 1142 2010-10-21 +Baltimore New York 2010-10-21 07:00:00 28.0 1599 2010-10-21 +Chicago New York 2010-10-21 07:00:00 142.0 361 2010-10-21 +Chicago New York 2010-10-21 07:00:00 77.0 897 2010-10-21 +Chicago New York 2010-10-21 07:00:00 53.0 1531 2010-10-21 +Chicago New York 2010-10-21 07:00:00 -5.0 1610 2010-10-21 +Chicago New York 2010-10-21 07:00:00 51.0 3198 2010-10-21 +Cleveland New York 2010-10-21 07:00:00 3.0 2630 2010-10-21 +Cleveland New York 2010-10-21 07:00:00 29.0 2646 2010-10-21 +Cleveland New York 2010-10-21 07:00:00 72.0 3014 2010-10-21 +Washington New York 2010-10-21 07:00:00 22.0 7291 2010-10-21 +Baltimore New York 2010-10-22 07:00:00 -12.0 1064 2010-10-22 +Baltimore New York 2010-10-22 07:00:00 54.0 1142 2010-10-22 +Baltimore New York 2010-10-22 07:00:00 18.0 1599 2010-10-22 +Chicago New York 2010-10-22 07:00:00 2.0 361 2010-10-22 +Chicago New York 2010-10-22 07:00:00 24.0 897 2010-10-22 +Chicago New York 2010-10-22 07:00:00 16.0 1531 2010-10-22 +Chicago New York 2010-10-22 07:00:00 -6.0 1610 2010-10-22 +Chicago New York 2010-10-22 07:00:00 -11.0 3198 2010-10-22 +Cleveland New York 2010-10-22 07:00:00 1.0 2630 2010-10-22 +Cleveland New York 2010-10-22 07:00:00 -25.0 2646 2010-10-22 +Cleveland New York 2010-10-22 07:00:00 -3.0 3014 2010-10-22 +Baltimore New York 2010-10-23 07:00:00 18.0 272 2010-10-23 +Baltimore New York 2010-10-23 07:00:00 -10.0 1805 2010-10-23 +Baltimore New York 2010-10-23 07:00:00 6.0 3171 2010-10-23 +Chicago New York 2010-10-23 07:00:00 3.0 384 2010-10-23 +Chicago New York 2010-10-23 07:00:00 32.0 426 2010-10-23 +Chicago New York 2010-10-23 07:00:00 1.0 650 2010-10-23 +Chicago New York 2010-10-23 07:00:00 11.0 3085 2010-10-23 +Cleveland New York 2010-10-23 07:00:00 -21.0 2932 2010-10-23 +Washington New York 2010-10-23 07:00:00 -25.0 5832 2010-10-23 +Washington New York 2010-10-23 07:00:00 -21.0 5904 2010-10-23 +Washington New York 2010-10-23 07:00:00 -18.0 5917 2010-10-23 +Washington New York 2010-10-23 07:00:00 -16.0 7274 2010-10-23 +Baltimore New York 2010-10-24 07:00:00 12.0 1599 2010-10-24 +Baltimore New York 2010-10-24 07:00:00 20.0 2571 2010-10-24 +Chicago New York 2010-10-24 07:00:00 10.0 361 2010-10-24 +Chicago New York 2010-10-24 07:00:00 113.0 897 2010-10-24 +Chicago New York 2010-10-24 07:00:00 -5.0 1531 2010-10-24 +Chicago New York 2010-10-24 07:00:00 -17.0 1610 2010-10-24 +Chicago New York 2010-10-24 07:00:00 -3.0 3198 2010-10-24 +Cleveland New York 2010-10-24 07:00:00 5.0 2254 2010-10-24 +Cleveland New York 2010-10-24 07:00:00 -11.0 2630 2010-10-24 +Cleveland New York 2010-10-24 07:00:00 -20.0 2646 2010-10-24 +Cleveland New York 2010-10-24 07:00:00 -9.0 3014 2010-10-24 +Washington New York 2010-10-24 07:00:00 -26.0 7282 2010-10-24 +Baltimore New York 2010-10-25 07:00:00 -25.0 1064 2010-10-25 +Baltimore New York 2010-10-25 07:00:00 92.0 1142 2010-10-25 +Baltimore New York 2010-10-25 07:00:00 106.0 1599 2010-10-25 +Chicago New York 2010-10-25 07:00:00 31.0 361 2010-10-25 +Chicago New York 2010-10-25 07:00:00 -1.0 897 2010-10-25 +Chicago New York 2010-10-25 07:00:00 43.0 1531 2010-10-25 +Chicago New York 2010-10-25 07:00:00 6.0 1610 2010-10-25 +Chicago New York 2010-10-25 07:00:00 -16.0 3198 2010-10-25 +Cleveland New York 2010-10-25 07:00:00 -4.0 2630 2010-10-25 +Cleveland New York 2010-10-25 07:00:00 81.0 2646 2010-10-25 +Cleveland New York 2010-10-25 07:00:00 42.0 3014 2010-10-25 +Washington New York 2010-10-25 07:00:00 9.0 7291 2010-10-25 +Baltimore New York 2010-10-26 07:00:00 -22.0 1064 2010-10-26 +Baltimore New York 2010-10-26 07:00:00 123.0 1142 2010-10-26 +Baltimore New York 2010-10-26 07:00:00 90.0 1599 2010-10-26 +Chicago New York 2010-10-26 07:00:00 12.0 361 2010-10-26 +Chicago New York 2010-10-26 07:00:00 0.0 897 2010-10-26 +Chicago New York 2010-10-26 07:00:00 29.0 1531 2010-10-26 +Chicago New York 2010-10-26 07:00:00 -17.0 1610 2010-10-26 +Chicago New York 2010-10-26 07:00:00 6.0 3198 2010-10-26 +Cleveland New York 2010-10-26 07:00:00 4.0 2630 2010-10-26 +Cleveland New York 2010-10-26 07:00:00 -27.0 2646 2010-10-26 +Cleveland New York 2010-10-26 07:00:00 -11.0 2662 2010-10-26 +Cleveland New York 2010-10-26 07:00:00 13.0 3014 2010-10-26 +Washington New York 2010-10-26 07:00:00 4.0 7291 2010-10-26 +Baltimore New York 2010-10-27 07:00:00 -18.0 1064 2010-10-27 +Baltimore New York 2010-10-27 07:00:00 49.0 1142 2010-10-27 +Baltimore New York 2010-10-27 07:00:00 92.0 1599 2010-10-27 +Chicago New York 2010-10-27 07:00:00 148.0 361 2010-10-27 +Chicago New York 2010-10-27 07:00:00 -11.0 897 2010-10-27 +Chicago New York 2010-10-27 07:00:00 70.0 1531 2010-10-27 +Chicago New York 2010-10-27 07:00:00 8.0 1610 2010-10-27 +Chicago New York 2010-10-27 07:00:00 21.0 3198 2010-10-27 +Cleveland New York 2010-10-27 07:00:00 16.0 2630 2010-10-27 +Cleveland New York 2010-10-27 07:00:00 27.0 3014 2010-10-27 +Washington New York 2010-10-27 07:00:00 26.0 7291 2010-10-27 +Baltimore New York 2010-10-28 07:00:00 -4.0 1064 2010-10-28 +Baltimore New York 2010-10-28 07:00:00 -14.0 1142 2010-10-28 +Baltimore New York 2010-10-28 07:00:00 -14.0 1599 2010-10-28 +Chicago New York 2010-10-28 07:00:00 2.0 361 2010-10-28 +Chicago New York 2010-10-28 07:00:00 2.0 897 2010-10-28 +Chicago New York 2010-10-28 07:00:00 -11.0 1531 2010-10-28 +Chicago New York 2010-10-28 07:00:00 3.0 1610 2010-10-28 +Chicago New York 2010-10-28 07:00:00 -18.0 3198 2010-10-28 +Cleveland New York 2010-10-28 07:00:00 3.0 2630 2010-10-28 +Cleveland New York 2010-10-28 07:00:00 -6.0 2646 2010-10-28 +Cleveland New York 2010-10-28 07:00:00 1.0 3014 2010-10-28 +Washington New York 2010-10-28 07:00:00 45.0 7291 2010-10-28 +Baltimore New York 2010-10-29 07:00:00 -24.0 1064 2010-10-29 +Baltimore New York 2010-10-29 07:00:00 21.0 1142 2010-10-29 +Baltimore New York 2010-10-29 07:00:00 -2.0 1599 2010-10-29 +Chicago New York 2010-10-29 07:00:00 -12.0 361 2010-10-29 +Chicago New York 2010-10-29 07:00:00 -11.0 897 2010-10-29 +Chicago New York 2010-10-29 07:00:00 15.0 1531 2010-10-29 +Chicago New York 2010-10-29 07:00:00 -18.0 1610 2010-10-29 +Chicago New York 2010-10-29 07:00:00 -4.0 3198 2010-10-29 +Cleveland New York 2010-10-29 07:00:00 -4.0 2630 2010-10-29 +Cleveland New York 2010-10-29 07:00:00 -19.0 2646 2010-10-29 +Cleveland New York 2010-10-29 07:00:00 -12.0 3014 2010-10-29 +Washington New York 2010-10-29 07:00:00 1.0 7291 2010-10-29 +Baltimore New York 2010-10-30 07:00:00 14.0 272 2010-10-30 +Baltimore New York 2010-10-30 07:00:00 -1.0 1805 2010-10-30 +Baltimore New York 2010-10-30 07:00:00 5.0 3171 2010-10-30 +Chicago New York 2010-10-30 07:00:00 -6.0 384 2010-10-30 +Chicago New York 2010-10-30 07:00:00 -10.0 426 2010-10-30 +Chicago New York 2010-10-30 07:00:00 -5.0 650 2010-10-30 +Chicago New York 2010-10-30 07:00:00 -5.0 3085 2010-10-30 +Cleveland New York 2010-10-30 07:00:00 -23.0 2018 2010-10-30 +Cleveland New York 2010-10-30 07:00:00 -12.0 2932 2010-10-30 +Washington New York 2010-10-30 07:00:00 -27.0 5904 2010-10-30 +Washington New York 2010-10-30 07:00:00 -16.0 5917 2010-10-30 +Baltimore New York 2010-10-31 07:00:00 -1.0 1599 2010-10-31 +Baltimore New York 2010-10-31 07:00:00 -14.0 2571 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -25.0 361 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -18.0 897 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -4.0 1531 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -22.0 1610 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -15.0 3198 2010-10-31 +Washington New York 2010-10-31 07:00:00 -18.0 7282 2010-10-31 +PREHOOK: query: select * from flights_tiny_orc_partitioned_date sort by fl_num, fl_date limit 25 +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc_partitioned_date +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 +#### A masked pattern was here #### +POSTHOOK: query: select * from flights_tiny_orc_partitioned_date sort by fl_num, fl_date limit 25 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 +#### A masked pattern was here #### +Baltimore New York 2010-10-23 07:00:00 18.0 272 2010-10-23 +Baltimore New York 2010-10-30 07:00:00 14.0 272 2010-10-30 +Chicago New York 2010-10-20 07:00:00 42.0 361 2010-10-20 +Chicago New York 2010-10-21 07:00:00 142.0 361 2010-10-21 +Chicago New York 2010-10-22 07:00:00 2.0 361 2010-10-22 +Chicago New York 2010-10-24 07:00:00 10.0 361 2010-10-24 +Chicago New York 2010-10-25 07:00:00 31.0 361 2010-10-25 +Chicago New York 2010-10-26 07:00:00 12.0 361 2010-10-26 +Chicago New York 2010-10-27 07:00:00 148.0 361 2010-10-27 +Chicago New York 2010-10-28 07:00:00 2.0 361 2010-10-28 +Chicago New York 2010-10-29 07:00:00 -12.0 361 2010-10-29 +Chicago New York 2010-10-31 07:00:00 -25.0 361 2010-10-31 +Chicago New York 2010-10-23 07:00:00 3.0 384 2010-10-23 +Chicago New York 2010-10-30 07:00:00 -6.0 384 2010-10-30 +Chicago New York 2010-10-23 07:00:00 32.0 426 2010-10-23 +Chicago New York 2010-10-30 07:00:00 -10.0 426 2010-10-30 +Chicago New York 2010-10-23 07:00:00 1.0 650 2010-10-23 +Chicago New York 2010-10-30 07:00:00 -5.0 650 2010-10-30 +Chicago New York 2010-10-20 07:00:00 24.0 897 2010-10-20 +Chicago New York 2010-10-21 07:00:00 77.0 897 2010-10-21 +Chicago New York 2010-10-22 07:00:00 24.0 897 2010-10-22 +Chicago New York 2010-10-24 07:00:00 113.0 897 2010-10-24 +Chicago New York 2010-10-25 07:00:00 -1.0 897 2010-10-25 +Chicago New York 2010-10-26 07:00:00 0.0 897 2010-10-26 +Chicago New York 2010-10-27 07:00:00 -11.0 897 2010-10-27 +PREHOOK: query: select fl_date, count(*) from flights_tiny_orc_partitioned_date group by fl_date +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc_partitioned_date +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 +#### A masked pattern was here #### +POSTHOOK: query: select fl_date, count(*) from flights_tiny_orc_partitioned_date group by fl_date +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 +#### A masked pattern was here #### +2010-10-20 11 +2010-10-21 12 +2010-10-22 11 +2010-10-23 12 +2010-10-24 12 +2010-10-25 12 +2010-10-26 13 +2010-10-27 11 +2010-10-28 12 +2010-10-29 12 +2010-10-30 11 +2010-10-31 8 +PREHOOK: query: explain +select * from flights_tiny_orc_partitioned_date +PREHOOK: type: QUERY +POSTHOOK: query: explain +select * from flights_tiny_orc_partitioned_date +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: flights_tiny_orc_partitioned_date + Select Operator + expressions: origin_city_name (type: string), dest_city_name (type: string), fl_time (type: timestamp), arr_delay (type: float), fl_num (type: int), fl_date (type: date) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + ListSink + +PREHOOK: query: select * from flights_tiny_orc_partitioned_date +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc_partitioned_date +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 +#### A masked pattern was here #### +POSTHOOK: query: select * from flights_tiny_orc_partitioned_date +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 +#### A masked pattern was here #### +Baltimore New York 2010-10-20 07:00:00 -30.0 1064 2010-10-20 +Baltimore New York 2010-10-20 07:00:00 23.0 1142 2010-10-20 +Baltimore New York 2010-10-20 07:00:00 6.0 1599 2010-10-20 +Chicago New York 2010-10-20 07:00:00 42.0 361 2010-10-20 +Chicago New York 2010-10-20 07:00:00 24.0 897 2010-10-20 +Chicago New York 2010-10-20 07:00:00 15.0 1531 2010-10-20 +Chicago New York 2010-10-20 07:00:00 -6.0 1610 2010-10-20 +Chicago New York 2010-10-20 07:00:00 -2.0 3198 2010-10-20 +Cleveland New York 2010-10-20 07:00:00 -8.0 2630 2010-10-20 +Cleveland New York 2010-10-20 07:00:00 -15.0 3014 2010-10-20 +Washington New York 2010-10-20 07:00:00 -2.0 7291 2010-10-20 +Baltimore New York 2010-10-21 07:00:00 17.0 1064 2010-10-21 +Baltimore New York 2010-10-21 07:00:00 105.0 1142 2010-10-21 +Baltimore New York 2010-10-21 07:00:00 28.0 1599 2010-10-21 +Chicago New York 2010-10-21 07:00:00 142.0 361 2010-10-21 +Chicago New York 2010-10-21 07:00:00 77.0 897 2010-10-21 +Chicago New York 2010-10-21 07:00:00 53.0 1531 2010-10-21 +Chicago New York 2010-10-21 07:00:00 -5.0 1610 2010-10-21 +Chicago New York 2010-10-21 07:00:00 51.0 3198 2010-10-21 +Cleveland New York 2010-10-21 07:00:00 3.0 2630 2010-10-21 +Cleveland New York 2010-10-21 07:00:00 29.0 2646 2010-10-21 +Cleveland New York 2010-10-21 07:00:00 72.0 3014 2010-10-21 +Washington New York 2010-10-21 07:00:00 22.0 7291 2010-10-21 +Baltimore New York 2010-10-22 07:00:00 -12.0 1064 2010-10-22 +Baltimore New York 2010-10-22 07:00:00 54.0 1142 2010-10-22 +Baltimore New York 2010-10-22 07:00:00 18.0 1599 2010-10-22 +Chicago New York 2010-10-22 07:00:00 2.0 361 2010-10-22 +Chicago New York 2010-10-22 07:00:00 24.0 897 2010-10-22 +Chicago New York 2010-10-22 07:00:00 16.0 1531 2010-10-22 +Chicago New York 2010-10-22 07:00:00 -6.0 1610 2010-10-22 +Chicago New York 2010-10-22 07:00:00 -11.0 3198 2010-10-22 +Cleveland New York 2010-10-22 07:00:00 1.0 2630 2010-10-22 +Cleveland New York 2010-10-22 07:00:00 -25.0 2646 2010-10-22 +Cleveland New York 2010-10-22 07:00:00 -3.0 3014 2010-10-22 +Baltimore New York 2010-10-23 07:00:00 18.0 272 2010-10-23 +Baltimore New York 2010-10-23 07:00:00 -10.0 1805 2010-10-23 +Baltimore New York 2010-10-23 07:00:00 6.0 3171 2010-10-23 +Chicago New York 2010-10-23 07:00:00 3.0 384 2010-10-23 +Chicago New York 2010-10-23 07:00:00 32.0 426 2010-10-23 +Chicago New York 2010-10-23 07:00:00 1.0 650 2010-10-23 +Chicago New York 2010-10-23 07:00:00 11.0 3085 2010-10-23 +Cleveland New York 2010-10-23 07:00:00 -21.0 2932 2010-10-23 +Washington New York 2010-10-23 07:00:00 -25.0 5832 2010-10-23 +Washington New York 2010-10-23 07:00:00 -21.0 5904 2010-10-23 +Washington New York 2010-10-23 07:00:00 -18.0 5917 2010-10-23 +Washington New York 2010-10-23 07:00:00 -16.0 7274 2010-10-23 +Baltimore New York 2010-10-24 07:00:00 12.0 1599 2010-10-24 +Baltimore New York 2010-10-24 07:00:00 20.0 2571 2010-10-24 +Chicago New York 2010-10-24 07:00:00 10.0 361 2010-10-24 +Chicago New York 2010-10-24 07:00:00 113.0 897 2010-10-24 +Chicago New York 2010-10-24 07:00:00 -5.0 1531 2010-10-24 +Chicago New York 2010-10-24 07:00:00 -17.0 1610 2010-10-24 +Chicago New York 2010-10-24 07:00:00 -3.0 3198 2010-10-24 +Cleveland New York 2010-10-24 07:00:00 5.0 2254 2010-10-24 +Cleveland New York 2010-10-24 07:00:00 -11.0 2630 2010-10-24 +Cleveland New York 2010-10-24 07:00:00 -20.0 2646 2010-10-24 +Cleveland New York 2010-10-24 07:00:00 -9.0 3014 2010-10-24 +Washington New York 2010-10-24 07:00:00 -26.0 7282 2010-10-24 +Baltimore New York 2010-10-25 07:00:00 -25.0 1064 2010-10-25 +Baltimore New York 2010-10-25 07:00:00 92.0 1142 2010-10-25 +Baltimore New York 2010-10-25 07:00:00 106.0 1599 2010-10-25 +Chicago New York 2010-10-25 07:00:00 31.0 361 2010-10-25 +Chicago New York 2010-10-25 07:00:00 -1.0 897 2010-10-25 +Chicago New York 2010-10-25 07:00:00 43.0 1531 2010-10-25 +Chicago New York 2010-10-25 07:00:00 6.0 1610 2010-10-25 +Chicago New York 2010-10-25 07:00:00 -16.0 3198 2010-10-25 +Cleveland New York 2010-10-25 07:00:00 -4.0 2630 2010-10-25 +Cleveland New York 2010-10-25 07:00:00 81.0 2646 2010-10-25 +Cleveland New York 2010-10-25 07:00:00 42.0 3014 2010-10-25 +Washington New York 2010-10-25 07:00:00 9.0 7291 2010-10-25 +Baltimore New York 2010-10-26 07:00:00 -22.0 1064 2010-10-26 +Baltimore New York 2010-10-26 07:00:00 123.0 1142 2010-10-26 +Baltimore New York 2010-10-26 07:00:00 90.0 1599 2010-10-26 +Chicago New York 2010-10-26 07:00:00 12.0 361 2010-10-26 +Chicago New York 2010-10-26 07:00:00 0.0 897 2010-10-26 +Chicago New York 2010-10-26 07:00:00 29.0 1531 2010-10-26 +Chicago New York 2010-10-26 07:00:00 -17.0 1610 2010-10-26 +Chicago New York 2010-10-26 07:00:00 6.0 3198 2010-10-26 +Cleveland New York 2010-10-26 07:00:00 4.0 2630 2010-10-26 +Cleveland New York 2010-10-26 07:00:00 -27.0 2646 2010-10-26 +Cleveland New York 2010-10-26 07:00:00 -11.0 2662 2010-10-26 +Cleveland New York 2010-10-26 07:00:00 13.0 3014 2010-10-26 +Washington New York 2010-10-26 07:00:00 4.0 7291 2010-10-26 +Baltimore New York 2010-10-27 07:00:00 -18.0 1064 2010-10-27 +Baltimore New York 2010-10-27 07:00:00 49.0 1142 2010-10-27 +Baltimore New York 2010-10-27 07:00:00 92.0 1599 2010-10-27 +Chicago New York 2010-10-27 07:00:00 148.0 361 2010-10-27 +Chicago New York 2010-10-27 07:00:00 -11.0 897 2010-10-27 +Chicago New York 2010-10-27 07:00:00 70.0 1531 2010-10-27 +Chicago New York 2010-10-27 07:00:00 8.0 1610 2010-10-27 +Chicago New York 2010-10-27 07:00:00 21.0 3198 2010-10-27 +Cleveland New York 2010-10-27 07:00:00 16.0 2630 2010-10-27 +Cleveland New York 2010-10-27 07:00:00 27.0 3014 2010-10-27 +Washington New York 2010-10-27 07:00:00 26.0 7291 2010-10-27 +Baltimore New York 2010-10-28 07:00:00 -4.0 1064 2010-10-28 +Baltimore New York 2010-10-28 07:00:00 -14.0 1142 2010-10-28 +Baltimore New York 2010-10-28 07:00:00 -14.0 1599 2010-10-28 +Chicago New York 2010-10-28 07:00:00 2.0 361 2010-10-28 +Chicago New York 2010-10-28 07:00:00 2.0 897 2010-10-28 +Chicago New York 2010-10-28 07:00:00 -11.0 1531 2010-10-28 +Chicago New York 2010-10-28 07:00:00 3.0 1610 2010-10-28 +Chicago New York 2010-10-28 07:00:00 -18.0 3198 2010-10-28 +Cleveland New York 2010-10-28 07:00:00 3.0 2630 2010-10-28 +Cleveland New York 2010-10-28 07:00:00 -6.0 2646 2010-10-28 +Cleveland New York 2010-10-28 07:00:00 1.0 3014 2010-10-28 +Washington New York 2010-10-28 07:00:00 45.0 7291 2010-10-28 +Baltimore New York 2010-10-29 07:00:00 -24.0 1064 2010-10-29 +Baltimore New York 2010-10-29 07:00:00 21.0 1142 2010-10-29 +Baltimore New York 2010-10-29 07:00:00 -2.0 1599 2010-10-29 +Chicago New York 2010-10-29 07:00:00 -12.0 361 2010-10-29 +Chicago New York 2010-10-29 07:00:00 -11.0 897 2010-10-29 +Chicago New York 2010-10-29 07:00:00 15.0 1531 2010-10-29 +Chicago New York 2010-10-29 07:00:00 -18.0 1610 2010-10-29 +Chicago New York 2010-10-29 07:00:00 -4.0 3198 2010-10-29 +Cleveland New York 2010-10-29 07:00:00 -4.0 2630 2010-10-29 +Cleveland New York 2010-10-29 07:00:00 -19.0 2646 2010-10-29 +Cleveland New York 2010-10-29 07:00:00 -12.0 3014 2010-10-29 +Washington New York 2010-10-29 07:00:00 1.0 7291 2010-10-29 +Baltimore New York 2010-10-30 07:00:00 14.0 272 2010-10-30 +Baltimore New York 2010-10-30 07:00:00 -1.0 1805 2010-10-30 +Baltimore New York 2010-10-30 07:00:00 5.0 3171 2010-10-30 +Chicago New York 2010-10-30 07:00:00 -6.0 384 2010-10-30 +Chicago New York 2010-10-30 07:00:00 -10.0 426 2010-10-30 +Chicago New York 2010-10-30 07:00:00 -5.0 650 2010-10-30 +Chicago New York 2010-10-30 07:00:00 -5.0 3085 2010-10-30 +Cleveland New York 2010-10-30 07:00:00 -23.0 2018 2010-10-30 +Cleveland New York 2010-10-30 07:00:00 -12.0 2932 2010-10-30 +Washington New York 2010-10-30 07:00:00 -27.0 5904 2010-10-30 +Washington New York 2010-10-30 07:00:00 -16.0 5917 2010-10-30 +Baltimore New York 2010-10-31 07:00:00 -1.0 1599 2010-10-31 +Baltimore New York 2010-10-31 07:00:00 -14.0 2571 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -25.0 361 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -18.0 897 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -4.0 1531 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -22.0 1610 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -15.0 3198 2010-10-31 +Washington New York 2010-10-31 07:00:00 -18.0 7282 2010-10-31 +PREHOOK: query: explain +select * from flights_tiny_orc_partitioned_date sort by fl_num, fl_date limit 25 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select * from flights_tiny_orc_partitioned_date sort by fl_num, fl_date limit 25 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: flights_tiny_orc_partitioned_date + Statistics: Num rows: 137 Data size: 31776 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), dest_city_name (type: string), fl_time (type: timestamp), arr_delay (type: float), fl_num (type: int), fl_date (type: date) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 137 Data size: 31776 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: int), _col5 (type: date) + sort order: ++ + Statistics: Num rows: 137 Data size: 31776 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: timestamp), _col3 (type: float) + Execution mode: vectorized + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: timestamp), VALUE._col3 (type: float), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: date) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 137 Data size: 31776 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 25 + Statistics: Num rows: 25 Data size: 5775 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: int), _col5 (type: date) + sort order: ++ + Statistics: Num rows: 25 Data size: 5775 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: timestamp), _col3 (type: float) + Execution mode: vectorized + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: timestamp), VALUE._col3 (type: float), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: date) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 25 Data size: 5775 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 25 + Statistics: Num rows: 25 Data size: 5775 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 25 Data size: 5775 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: 25 + Processor Tree: + ListSink + +PREHOOK: query: select * from flights_tiny_orc_partitioned_date sort by fl_num, fl_date limit 25 +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc_partitioned_date +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 +#### A masked pattern was here #### +POSTHOOK: query: select * from flights_tiny_orc_partitioned_date sort by fl_num, fl_date limit 25 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 +#### A masked pattern was here #### +Baltimore New York 2010-10-23 07:00:00 18.0 272 2010-10-23 +Baltimore New York 2010-10-30 07:00:00 14.0 272 2010-10-30 +Chicago New York 2010-10-20 07:00:00 42.0 361 2010-10-20 +Chicago New York 2010-10-21 07:00:00 142.0 361 2010-10-21 +Chicago New York 2010-10-22 07:00:00 2.0 361 2010-10-22 +Chicago New York 2010-10-24 07:00:00 10.0 361 2010-10-24 +Chicago New York 2010-10-25 07:00:00 31.0 361 2010-10-25 +Chicago New York 2010-10-26 07:00:00 12.0 361 2010-10-26 +Chicago New York 2010-10-27 07:00:00 148.0 361 2010-10-27 +Chicago New York 2010-10-28 07:00:00 2.0 361 2010-10-28 +Chicago New York 2010-10-29 07:00:00 -12.0 361 2010-10-29 +Chicago New York 2010-10-31 07:00:00 -25.0 361 2010-10-31 +Chicago New York 2010-10-23 07:00:00 3.0 384 2010-10-23 +Chicago New York 2010-10-30 07:00:00 -6.0 384 2010-10-30 +Chicago New York 2010-10-23 07:00:00 32.0 426 2010-10-23 +Chicago New York 2010-10-30 07:00:00 -10.0 426 2010-10-30 +Chicago New York 2010-10-23 07:00:00 1.0 650 2010-10-23 +Chicago New York 2010-10-30 07:00:00 -5.0 650 2010-10-30 +Chicago New York 2010-10-20 07:00:00 24.0 897 2010-10-20 +Chicago New York 2010-10-21 07:00:00 77.0 897 2010-10-21 +Chicago New York 2010-10-22 07:00:00 24.0 897 2010-10-22 +Chicago New York 2010-10-24 07:00:00 113.0 897 2010-10-24 +Chicago New York 2010-10-25 07:00:00 -1.0 897 2010-10-25 +Chicago New York 2010-10-26 07:00:00 0.0 897 2010-10-26 +Chicago New York 2010-10-27 07:00:00 -11.0 897 2010-10-27 +PREHOOK: query: explain +select fl_date, count(*) from flights_tiny_orc_partitioned_date group by fl_date +PREHOOK: type: QUERY +POSTHOOK: query: explain +select fl_date, count(*) from flights_tiny_orc_partitioned_date group by fl_date +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: flights_tiny_orc_partitioned_date + Statistics: Num rows: 137 Data size: 31776 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: fl_date (type: date) + outputColumnNames: fl_date + Statistics: Num rows: 137 Data size: 31776 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: fl_date (type: date) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 137 Data size: 31776 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: date) + sort order: + + Map-reduce partition columns: _col0 (type: date) + Statistics: Num rows: 137 Data size: 31776 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: vectorized + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: date) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 68 Data size: 15772 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: date), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 68 Data size: 15772 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 68 Data size: 15772 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select fl_date, count(*) from flights_tiny_orc_partitioned_date group by fl_date +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc_partitioned_date +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 +#### A masked pattern was here #### +POSTHOOK: query: select fl_date, count(*) from flights_tiny_orc_partitioned_date group by fl_date +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 +#### A masked pattern was here #### +2010-10-20 11 +2010-10-21 12 +2010-10-22 11 +2010-10-23 12 +2010-10-24 12 +2010-10-25 12 +2010-10-26 13 +2010-10-27 11 +2010-10-28 12 +2010-10-29 12 +2010-10-30 11 +2010-10-31 8 +PREHOOK: query: CREATE TABLE flights_tiny_orc_partitioned_timestamp ( + origin_city_name STRING, + dest_city_name STRING, + fl_date DATE, + arr_delay FLOAT, + fl_num INT +) +PARTITIONED BY (fl_time TIMESTAMP) +STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@flights_tiny_orc_partitioned_timestamp +POSTHOOK: query: CREATE TABLE flights_tiny_orc_partitioned_timestamp ( + origin_city_name STRING, + dest_city_name STRING, + fl_date DATE, + arr_delay FLOAT, + fl_num INT +) +PARTITIONED BY (fl_time TIMESTAMP) +STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp +PREHOOK: query: INSERT INTO TABLE flights_tiny_orc_partitioned_timestamp +PARTITION (fl_time) +SELECT origin_city_name, dest_city_name, fl_date, arr_delay, fl_num, fl_time +FROM flights_tiny_orc +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc +PREHOOK: Output: default@flights_tiny_orc_partitioned_timestamp +POSTHOOK: query: INSERT INTO TABLE flights_tiny_orc_partitioned_timestamp +PARTITION (fl_time) +SELECT origin_city_name, dest_city_name, fl_date, arr_delay, fl_num, fl_time +FROM flights_tiny_orc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc +POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20 07%3A00%3A00 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21 07%3A00%3A00 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22 07%3A00%3A00 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23 07%3A00%3A00 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24 07%3A00%3A00 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25 07%3A00%3A00 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26 07%3A00%3A00 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27 07%3A00%3A00 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28 07%3A00%3A00 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29 07%3A00%3A00 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-20 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-20 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-20 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-20 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-20 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-21 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-21 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-21 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-21 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-21 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-22 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-22 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-22 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-22 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-22 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-23 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-23 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-23 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-23 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-23 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-24 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-24 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-24 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-24 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-24 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-25 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-25 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-25 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-25 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-25 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-26 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-26 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-26 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-26 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-26 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-27 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-27 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-27 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-27 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-27 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-28 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-28 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-28 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-28 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-28 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-29 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-29 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-29 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-29 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-29 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-30 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-30 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-30 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-30 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-30 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-31 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-31 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-31 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-31 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-31 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +PREHOOK: query: select * from flights_tiny_orc_partitioned_timestamp +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 +#### A masked pattern was here #### +POSTHOOK: query: select * from flights_tiny_orc_partitioned_timestamp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 +#### A masked pattern was here #### +Baltimore New York 2010-10-20 -30.0 1064 2010-10-20 07:00:00 +Baltimore New York 2010-10-20 23.0 1142 2010-10-20 07:00:00 +Baltimore New York 2010-10-20 6.0 1599 2010-10-20 07:00:00 +Chicago New York 2010-10-20 42.0 361 2010-10-20 07:00:00 +Chicago New York 2010-10-20 24.0 897 2010-10-20 07:00:00 +Chicago New York 2010-10-20 15.0 1531 2010-10-20 07:00:00 +Chicago New York 2010-10-20 -6.0 1610 2010-10-20 07:00:00 +Chicago New York 2010-10-20 -2.0 3198 2010-10-20 07:00:00 +Cleveland New York 2010-10-20 -8.0 2630 2010-10-20 07:00:00 +Cleveland New York 2010-10-20 -15.0 3014 2010-10-20 07:00:00 +Washington New York 2010-10-20 -2.0 7291 2010-10-20 07:00:00 +Baltimore New York 2010-10-21 17.0 1064 2010-10-21 07:00:00 +Baltimore New York 2010-10-21 105.0 1142 2010-10-21 07:00:00 +Baltimore New York 2010-10-21 28.0 1599 2010-10-21 07:00:00 +Chicago New York 2010-10-21 142.0 361 2010-10-21 07:00:00 +Chicago New York 2010-10-21 77.0 897 2010-10-21 07:00:00 +Chicago New York 2010-10-21 53.0 1531 2010-10-21 07:00:00 +Chicago New York 2010-10-21 -5.0 1610 2010-10-21 07:00:00 +Chicago New York 2010-10-21 51.0 3198 2010-10-21 07:00:00 +Cleveland New York 2010-10-21 3.0 2630 2010-10-21 07:00:00 +Cleveland New York 2010-10-21 29.0 2646 2010-10-21 07:00:00 +Cleveland New York 2010-10-21 72.0 3014 2010-10-21 07:00:00 +Washington New York 2010-10-21 22.0 7291 2010-10-21 07:00:00 +Baltimore New York 2010-10-22 -12.0 1064 2010-10-22 07:00:00 +Baltimore New York 2010-10-22 54.0 1142 2010-10-22 07:00:00 +Baltimore New York 2010-10-22 18.0 1599 2010-10-22 07:00:00 +Chicago New York 2010-10-22 2.0 361 2010-10-22 07:00:00 +Chicago New York 2010-10-22 24.0 897 2010-10-22 07:00:00 +Chicago New York 2010-10-22 16.0 1531 2010-10-22 07:00:00 +Chicago New York 2010-10-22 -6.0 1610 2010-10-22 07:00:00 +Chicago New York 2010-10-22 -11.0 3198 2010-10-22 07:00:00 +Cleveland New York 2010-10-22 1.0 2630 2010-10-22 07:00:00 +Cleveland New York 2010-10-22 -25.0 2646 2010-10-22 07:00:00 +Cleveland New York 2010-10-22 -3.0 3014 2010-10-22 07:00:00 +Baltimore New York 2010-10-23 18.0 272 2010-10-23 07:00:00 +Baltimore New York 2010-10-23 -10.0 1805 2010-10-23 07:00:00 +Baltimore New York 2010-10-23 6.0 3171 2010-10-23 07:00:00 +Chicago New York 2010-10-23 3.0 384 2010-10-23 07:00:00 +Chicago New York 2010-10-23 32.0 426 2010-10-23 07:00:00 +Chicago New York 2010-10-23 1.0 650 2010-10-23 07:00:00 +Chicago New York 2010-10-23 11.0 3085 2010-10-23 07:00:00 +Cleveland New York 2010-10-23 -21.0 2932 2010-10-23 07:00:00 +Washington New York 2010-10-23 -25.0 5832 2010-10-23 07:00:00 +Washington New York 2010-10-23 -21.0 5904 2010-10-23 07:00:00 +Washington New York 2010-10-23 -18.0 5917 2010-10-23 07:00:00 +Washington New York 2010-10-23 -16.0 7274 2010-10-23 07:00:00 +Baltimore New York 2010-10-24 12.0 1599 2010-10-24 07:00:00 +Baltimore New York 2010-10-24 20.0 2571 2010-10-24 07:00:00 +Chicago New York 2010-10-24 10.0 361 2010-10-24 07:00:00 +Chicago New York 2010-10-24 113.0 897 2010-10-24 07:00:00 +Chicago New York 2010-10-24 -5.0 1531 2010-10-24 07:00:00 +Chicago New York 2010-10-24 -17.0 1610 2010-10-24 07:00:00 +Chicago New York 2010-10-24 -3.0 3198 2010-10-24 07:00:00 +Cleveland New York 2010-10-24 5.0 2254 2010-10-24 07:00:00 +Cleveland New York 2010-10-24 -11.0 2630 2010-10-24 07:00:00 +Cleveland New York 2010-10-24 -20.0 2646 2010-10-24 07:00:00 +Cleveland New York 2010-10-24 -9.0 3014 2010-10-24 07:00:00 +Washington New York 2010-10-24 -26.0 7282 2010-10-24 07:00:00 +Baltimore New York 2010-10-25 -25.0 1064 2010-10-25 07:00:00 +Baltimore New York 2010-10-25 92.0 1142 2010-10-25 07:00:00 +Baltimore New York 2010-10-25 106.0 1599 2010-10-25 07:00:00 +Chicago New York 2010-10-25 31.0 361 2010-10-25 07:00:00 +Chicago New York 2010-10-25 -1.0 897 2010-10-25 07:00:00 +Chicago New York 2010-10-25 43.0 1531 2010-10-25 07:00:00 +Chicago New York 2010-10-25 6.0 1610 2010-10-25 07:00:00 +Chicago New York 2010-10-25 -16.0 3198 2010-10-25 07:00:00 +Cleveland New York 2010-10-25 -4.0 2630 2010-10-25 07:00:00 +Cleveland New York 2010-10-25 81.0 2646 2010-10-25 07:00:00 +Cleveland New York 2010-10-25 42.0 3014 2010-10-25 07:00:00 +Washington New York 2010-10-25 9.0 7291 2010-10-25 07:00:00 +Baltimore New York 2010-10-26 -22.0 1064 2010-10-26 07:00:00 +Baltimore New York 2010-10-26 123.0 1142 2010-10-26 07:00:00 +Baltimore New York 2010-10-26 90.0 1599 2010-10-26 07:00:00 +Chicago New York 2010-10-26 12.0 361 2010-10-26 07:00:00 +Chicago New York 2010-10-26 0.0 897 2010-10-26 07:00:00 +Chicago New York 2010-10-26 29.0 1531 2010-10-26 07:00:00 +Chicago New York 2010-10-26 -17.0 1610 2010-10-26 07:00:00 +Chicago New York 2010-10-26 6.0 3198 2010-10-26 07:00:00 +Cleveland New York 2010-10-26 4.0 2630 2010-10-26 07:00:00 +Cleveland New York 2010-10-26 -27.0 2646 2010-10-26 07:00:00 +Cleveland New York 2010-10-26 -11.0 2662 2010-10-26 07:00:00 +Cleveland New York 2010-10-26 13.0 3014 2010-10-26 07:00:00 +Washington New York 2010-10-26 4.0 7291 2010-10-26 07:00:00 +Baltimore New York 2010-10-27 -18.0 1064 2010-10-27 07:00:00 +Baltimore New York 2010-10-27 49.0 1142 2010-10-27 07:00:00 +Baltimore New York 2010-10-27 92.0 1599 2010-10-27 07:00:00 +Chicago New York 2010-10-27 148.0 361 2010-10-27 07:00:00 +Chicago New York 2010-10-27 -11.0 897 2010-10-27 07:00:00 +Chicago New York 2010-10-27 70.0 1531 2010-10-27 07:00:00 +Chicago New York 2010-10-27 8.0 1610 2010-10-27 07:00:00 +Chicago New York 2010-10-27 21.0 3198 2010-10-27 07:00:00 +Cleveland New York 2010-10-27 16.0 2630 2010-10-27 07:00:00 +Cleveland New York 2010-10-27 27.0 3014 2010-10-27 07:00:00 +Washington New York 2010-10-27 26.0 7291 2010-10-27 07:00:00 +Baltimore New York 2010-10-28 -4.0 1064 2010-10-28 07:00:00 +Baltimore New York 2010-10-28 -14.0 1142 2010-10-28 07:00:00 +Baltimore New York 2010-10-28 -14.0 1599 2010-10-28 07:00:00 +Chicago New York 2010-10-28 2.0 361 2010-10-28 07:00:00 +Chicago New York 2010-10-28 2.0 897 2010-10-28 07:00:00 +Chicago New York 2010-10-28 -11.0 1531 2010-10-28 07:00:00 +Chicago New York 2010-10-28 3.0 1610 2010-10-28 07:00:00 +Chicago New York 2010-10-28 -18.0 3198 2010-10-28 07:00:00 +Cleveland New York 2010-10-28 3.0 2630 2010-10-28 07:00:00 +Cleveland New York 2010-10-28 -6.0 2646 2010-10-28 07:00:00 +Cleveland New York 2010-10-28 1.0 3014 2010-10-28 07:00:00 +Washington New York 2010-10-28 45.0 7291 2010-10-28 07:00:00 +Baltimore New York 2010-10-29 -24.0 1064 2010-10-29 07:00:00 +Baltimore New York 2010-10-29 21.0 1142 2010-10-29 07:00:00 +Baltimore New York 2010-10-29 -2.0 1599 2010-10-29 07:00:00 +Chicago New York 2010-10-29 -12.0 361 2010-10-29 07:00:00 +Chicago New York 2010-10-29 -11.0 897 2010-10-29 07:00:00 +Chicago New York 2010-10-29 15.0 1531 2010-10-29 07:00:00 +Chicago New York 2010-10-29 -18.0 1610 2010-10-29 07:00:00 +Chicago New York 2010-10-29 -4.0 3198 2010-10-29 07:00:00 +Cleveland New York 2010-10-29 -4.0 2630 2010-10-29 07:00:00 +Cleveland New York 2010-10-29 -19.0 2646 2010-10-29 07:00:00 +Cleveland New York 2010-10-29 -12.0 3014 2010-10-29 07:00:00 +Washington New York 2010-10-29 1.0 7291 2010-10-29 07:00:00 +Baltimore New York 2010-10-30 14.0 272 2010-10-30 07:00:00 +Baltimore New York 2010-10-30 -1.0 1805 2010-10-30 07:00:00 +Baltimore New York 2010-10-30 5.0 3171 2010-10-30 07:00:00 +Chicago New York 2010-10-30 -6.0 384 2010-10-30 07:00:00 +Chicago New York 2010-10-30 -10.0 426 2010-10-30 07:00:00 +Chicago New York 2010-10-30 -5.0 650 2010-10-30 07:00:00 +Chicago New York 2010-10-30 -5.0 3085 2010-10-30 07:00:00 +Cleveland New York 2010-10-30 -23.0 2018 2010-10-30 07:00:00 +Cleveland New York 2010-10-30 -12.0 2932 2010-10-30 07:00:00 +Washington New York 2010-10-30 -27.0 5904 2010-10-30 07:00:00 +Washington New York 2010-10-30 -16.0 5917 2010-10-30 07:00:00 +Baltimore New York 2010-10-31 -1.0 1599 2010-10-31 07:00:00 +Baltimore New York 2010-10-31 -14.0 2571 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -25.0 361 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -18.0 897 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -4.0 1531 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -22.0 1610 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -15.0 3198 2010-10-31 07:00:00 +Washington New York 2010-10-31 -18.0 7282 2010-10-31 07:00:00 +PREHOOK: query: select * from flights_tiny_orc_partitioned_timestamp sort by fl_num, fl_time limit 25 +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 +#### A masked pattern was here #### +POSTHOOK: query: select * from flights_tiny_orc_partitioned_timestamp sort by fl_num, fl_time limit 25 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 +#### A masked pattern was here #### +Baltimore New York 2010-10-23 18.0 272 2010-10-23 07:00:00 +Baltimore New York 2010-10-30 14.0 272 2010-10-30 07:00:00 +Chicago New York 2010-10-20 42.0 361 2010-10-20 07:00:00 +Chicago New York 2010-10-21 142.0 361 2010-10-21 07:00:00 +Chicago New York 2010-10-22 2.0 361 2010-10-22 07:00:00 +Chicago New York 2010-10-24 10.0 361 2010-10-24 07:00:00 +Chicago New York 2010-10-25 31.0 361 2010-10-25 07:00:00 +Chicago New York 2010-10-26 12.0 361 2010-10-26 07:00:00 +Chicago New York 2010-10-27 148.0 361 2010-10-27 07:00:00 +Chicago New York 2010-10-28 2.0 361 2010-10-28 07:00:00 +Chicago New York 2010-10-29 -12.0 361 2010-10-29 07:00:00 +Chicago New York 2010-10-31 -25.0 361 2010-10-31 07:00:00 +Chicago New York 2010-10-23 3.0 384 2010-10-23 07:00:00 +Chicago New York 2010-10-30 -6.0 384 2010-10-30 07:00:00 +Chicago New York 2010-10-23 32.0 426 2010-10-23 07:00:00 +Chicago New York 2010-10-30 -10.0 426 2010-10-30 07:00:00 +Chicago New York 2010-10-23 1.0 650 2010-10-23 07:00:00 +Chicago New York 2010-10-30 -5.0 650 2010-10-30 07:00:00 +Chicago New York 2010-10-20 24.0 897 2010-10-20 07:00:00 +Chicago New York 2010-10-21 77.0 897 2010-10-21 07:00:00 +Chicago New York 2010-10-22 24.0 897 2010-10-22 07:00:00 +Chicago New York 2010-10-24 113.0 897 2010-10-24 07:00:00 +Chicago New York 2010-10-25 -1.0 897 2010-10-25 07:00:00 +Chicago New York 2010-10-26 0.0 897 2010-10-26 07:00:00 +Chicago New York 2010-10-27 -11.0 897 2010-10-27 07:00:00 +PREHOOK: query: select fl_time, count(*) from flights_tiny_orc_partitioned_timestamp group by fl_time +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 +#### A masked pattern was here #### +POSTHOOK: query: select fl_time, count(*) from flights_tiny_orc_partitioned_timestamp group by fl_time +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 +#### A masked pattern was here #### +2010-10-20 07:00:00 11 +2010-10-21 07:00:00 12 +2010-10-22 07:00:00 11 +2010-10-23 07:00:00 12 +2010-10-24 07:00:00 12 +2010-10-25 07:00:00 12 +2010-10-26 07:00:00 13 +2010-10-27 07:00:00 11 +2010-10-28 07:00:00 12 +2010-10-29 07:00:00 12 +2010-10-30 07:00:00 11 +2010-10-31 07:00:00 8 +PREHOOK: query: explain +select * from flights_tiny_orc_partitioned_timestamp +PREHOOK: type: QUERY +POSTHOOK: query: explain +select * from flights_tiny_orc_partitioned_timestamp +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: flights_tiny_orc_partitioned_timestamp + Select Operator + expressions: origin_city_name (type: string), dest_city_name (type: string), fl_date (type: date), arr_delay (type: float), fl_num (type: int), fl_time (type: timestamp) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + ListSink + +PREHOOK: query: select * from flights_tiny_orc_partitioned_timestamp +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 +#### A masked pattern was here #### +POSTHOOK: query: select * from flights_tiny_orc_partitioned_timestamp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 +#### A masked pattern was here #### +Baltimore New York 2010-10-20 -30.0 1064 2010-10-20 07:00:00 +Baltimore New York 2010-10-20 23.0 1142 2010-10-20 07:00:00 +Baltimore New York 2010-10-20 6.0 1599 2010-10-20 07:00:00 +Chicago New York 2010-10-20 42.0 361 2010-10-20 07:00:00 +Chicago New York 2010-10-20 24.0 897 2010-10-20 07:00:00 +Chicago New York 2010-10-20 15.0 1531 2010-10-20 07:00:00 +Chicago New York 2010-10-20 -6.0 1610 2010-10-20 07:00:00 +Chicago New York 2010-10-20 -2.0 3198 2010-10-20 07:00:00 +Cleveland New York 2010-10-20 -8.0 2630 2010-10-20 07:00:00 +Cleveland New York 2010-10-20 -15.0 3014 2010-10-20 07:00:00 +Washington New York 2010-10-20 -2.0 7291 2010-10-20 07:00:00 +Baltimore New York 2010-10-21 17.0 1064 2010-10-21 07:00:00 +Baltimore New York 2010-10-21 105.0 1142 2010-10-21 07:00:00 +Baltimore New York 2010-10-21 28.0 1599 2010-10-21 07:00:00 +Chicago New York 2010-10-21 142.0 361 2010-10-21 07:00:00 +Chicago New York 2010-10-21 77.0 897 2010-10-21 07:00:00 +Chicago New York 2010-10-21 53.0 1531 2010-10-21 07:00:00 +Chicago New York 2010-10-21 -5.0 1610 2010-10-21 07:00:00 +Chicago New York 2010-10-21 51.0 3198 2010-10-21 07:00:00 +Cleveland New York 2010-10-21 3.0 2630 2010-10-21 07:00:00 +Cleveland New York 2010-10-21 29.0 2646 2010-10-21 07:00:00 +Cleveland New York 2010-10-21 72.0 3014 2010-10-21 07:00:00 +Washington New York 2010-10-21 22.0 7291 2010-10-21 07:00:00 +Baltimore New York 2010-10-22 -12.0 1064 2010-10-22 07:00:00 +Baltimore New York 2010-10-22 54.0 1142 2010-10-22 07:00:00 +Baltimore New York 2010-10-22 18.0 1599 2010-10-22 07:00:00 +Chicago New York 2010-10-22 2.0 361 2010-10-22 07:00:00 +Chicago New York 2010-10-22 24.0 897 2010-10-22 07:00:00 +Chicago New York 2010-10-22 16.0 1531 2010-10-22 07:00:00 +Chicago New York 2010-10-22 -6.0 1610 2010-10-22 07:00:00 +Chicago New York 2010-10-22 -11.0 3198 2010-10-22 07:00:00 +Cleveland New York 2010-10-22 1.0 2630 2010-10-22 07:00:00 +Cleveland New York 2010-10-22 -25.0 2646 2010-10-22 07:00:00 +Cleveland New York 2010-10-22 -3.0 3014 2010-10-22 07:00:00 +Baltimore New York 2010-10-23 18.0 272 2010-10-23 07:00:00 +Baltimore New York 2010-10-23 -10.0 1805 2010-10-23 07:00:00 +Baltimore New York 2010-10-23 6.0 3171 2010-10-23 07:00:00 +Chicago New York 2010-10-23 3.0 384 2010-10-23 07:00:00 +Chicago New York 2010-10-23 32.0 426 2010-10-23 07:00:00 +Chicago New York 2010-10-23 1.0 650 2010-10-23 07:00:00 +Chicago New York 2010-10-23 11.0 3085 2010-10-23 07:00:00 +Cleveland New York 2010-10-23 -21.0 2932 2010-10-23 07:00:00 +Washington New York 2010-10-23 -25.0 5832 2010-10-23 07:00:00 +Washington New York 2010-10-23 -21.0 5904 2010-10-23 07:00:00 +Washington New York 2010-10-23 -18.0 5917 2010-10-23 07:00:00 +Washington New York 2010-10-23 -16.0 7274 2010-10-23 07:00:00 +Baltimore New York 2010-10-24 12.0 1599 2010-10-24 07:00:00 +Baltimore New York 2010-10-24 20.0 2571 2010-10-24 07:00:00 +Chicago New York 2010-10-24 10.0 361 2010-10-24 07:00:00 +Chicago New York 2010-10-24 113.0 897 2010-10-24 07:00:00 +Chicago New York 2010-10-24 -5.0 1531 2010-10-24 07:00:00 +Chicago New York 2010-10-24 -17.0 1610 2010-10-24 07:00:00 +Chicago New York 2010-10-24 -3.0 3198 2010-10-24 07:00:00 +Cleveland New York 2010-10-24 5.0 2254 2010-10-24 07:00:00 +Cleveland New York 2010-10-24 -11.0 2630 2010-10-24 07:00:00 +Cleveland New York 2010-10-24 -20.0 2646 2010-10-24 07:00:00 +Cleveland New York 2010-10-24 -9.0 3014 2010-10-24 07:00:00 +Washington New York 2010-10-24 -26.0 7282 2010-10-24 07:00:00 +Baltimore New York 2010-10-25 -25.0 1064 2010-10-25 07:00:00 +Baltimore New York 2010-10-25 92.0 1142 2010-10-25 07:00:00 +Baltimore New York 2010-10-25 106.0 1599 2010-10-25 07:00:00 +Chicago New York 2010-10-25 31.0 361 2010-10-25 07:00:00 +Chicago New York 2010-10-25 -1.0 897 2010-10-25 07:00:00 +Chicago New York 2010-10-25 43.0 1531 2010-10-25 07:00:00 +Chicago New York 2010-10-25 6.0 1610 2010-10-25 07:00:00 +Chicago New York 2010-10-25 -16.0 3198 2010-10-25 07:00:00 +Cleveland New York 2010-10-25 -4.0 2630 2010-10-25 07:00:00 +Cleveland New York 2010-10-25 81.0 2646 2010-10-25 07:00:00 +Cleveland New York 2010-10-25 42.0 3014 2010-10-25 07:00:00 +Washington New York 2010-10-25 9.0 7291 2010-10-25 07:00:00 +Baltimore New York 2010-10-26 -22.0 1064 2010-10-26 07:00:00 +Baltimore New York 2010-10-26 123.0 1142 2010-10-26 07:00:00 +Baltimore New York 2010-10-26 90.0 1599 2010-10-26 07:00:00 +Chicago New York 2010-10-26 12.0 361 2010-10-26 07:00:00 +Chicago New York 2010-10-26 0.0 897 2010-10-26 07:00:00 +Chicago New York 2010-10-26 29.0 1531 2010-10-26 07:00:00 +Chicago New York 2010-10-26 -17.0 1610 2010-10-26 07:00:00 +Chicago New York 2010-10-26 6.0 3198 2010-10-26 07:00:00 +Cleveland New York 2010-10-26 4.0 2630 2010-10-26 07:00:00 +Cleveland New York 2010-10-26 -27.0 2646 2010-10-26 07:00:00 +Cleveland New York 2010-10-26 -11.0 2662 2010-10-26 07:00:00 +Cleveland New York 2010-10-26 13.0 3014 2010-10-26 07:00:00 +Washington New York 2010-10-26 4.0 7291 2010-10-26 07:00:00 +Baltimore New York 2010-10-27 -18.0 1064 2010-10-27 07:00:00 +Baltimore New York 2010-10-27 49.0 1142 2010-10-27 07:00:00 +Baltimore New York 2010-10-27 92.0 1599 2010-10-27 07:00:00 +Chicago New York 2010-10-27 148.0 361 2010-10-27 07:00:00 +Chicago New York 2010-10-27 -11.0 897 2010-10-27 07:00:00 +Chicago New York 2010-10-27 70.0 1531 2010-10-27 07:00:00 +Chicago New York 2010-10-27 8.0 1610 2010-10-27 07:00:00 +Chicago New York 2010-10-27 21.0 3198 2010-10-27 07:00:00 +Cleveland New York 2010-10-27 16.0 2630 2010-10-27 07:00:00 +Cleveland New York 2010-10-27 27.0 3014 2010-10-27 07:00:00 +Washington New York 2010-10-27 26.0 7291 2010-10-27 07:00:00 +Baltimore New York 2010-10-28 -4.0 1064 2010-10-28 07:00:00 +Baltimore New York 2010-10-28 -14.0 1142 2010-10-28 07:00:00 +Baltimore New York 2010-10-28 -14.0 1599 2010-10-28 07:00:00 +Chicago New York 2010-10-28 2.0 361 2010-10-28 07:00:00 +Chicago New York 2010-10-28 2.0 897 2010-10-28 07:00:00 +Chicago New York 2010-10-28 -11.0 1531 2010-10-28 07:00:00 +Chicago New York 2010-10-28 3.0 1610 2010-10-28 07:00:00 +Chicago New York 2010-10-28 -18.0 3198 2010-10-28 07:00:00 +Cleveland New York 2010-10-28 3.0 2630 2010-10-28 07:00:00 +Cleveland New York 2010-10-28 -6.0 2646 2010-10-28 07:00:00 +Cleveland New York 2010-10-28 1.0 3014 2010-10-28 07:00:00 +Washington New York 2010-10-28 45.0 7291 2010-10-28 07:00:00 +Baltimore New York 2010-10-29 -24.0 1064 2010-10-29 07:00:00 +Baltimore New York 2010-10-29 21.0 1142 2010-10-29 07:00:00 +Baltimore New York 2010-10-29 -2.0 1599 2010-10-29 07:00:00 +Chicago New York 2010-10-29 -12.0 361 2010-10-29 07:00:00 +Chicago New York 2010-10-29 -11.0 897 2010-10-29 07:00:00 +Chicago New York 2010-10-29 15.0 1531 2010-10-29 07:00:00 +Chicago New York 2010-10-29 -18.0 1610 2010-10-29 07:00:00 +Chicago New York 2010-10-29 -4.0 3198 2010-10-29 07:00:00 +Cleveland New York 2010-10-29 -4.0 2630 2010-10-29 07:00:00 +Cleveland New York 2010-10-29 -19.0 2646 2010-10-29 07:00:00 +Cleveland New York 2010-10-29 -12.0 3014 2010-10-29 07:00:00 +Washington New York 2010-10-29 1.0 7291 2010-10-29 07:00:00 +Baltimore New York 2010-10-30 14.0 272 2010-10-30 07:00:00 +Baltimore New York 2010-10-30 -1.0 1805 2010-10-30 07:00:00 +Baltimore New York 2010-10-30 5.0 3171 2010-10-30 07:00:00 +Chicago New York 2010-10-30 -6.0 384 2010-10-30 07:00:00 +Chicago New York 2010-10-30 -10.0 426 2010-10-30 07:00:00 +Chicago New York 2010-10-30 -5.0 650 2010-10-30 07:00:00 +Chicago New York 2010-10-30 -5.0 3085 2010-10-30 07:00:00 +Cleveland New York 2010-10-30 -23.0 2018 2010-10-30 07:00:00 +Cleveland New York 2010-10-30 -12.0 2932 2010-10-30 07:00:00 +Washington New York 2010-10-30 -27.0 5904 2010-10-30 07:00:00 +Washington New York 2010-10-30 -16.0 5917 2010-10-30 07:00:00 +Baltimore New York 2010-10-31 -1.0 1599 2010-10-31 07:00:00 +Baltimore New York 2010-10-31 -14.0 2571 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -25.0 361 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -18.0 897 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -4.0 1531 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -22.0 1610 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -15.0 3198 2010-10-31 07:00:00 +Washington New York 2010-10-31 -18.0 7282 2010-10-31 07:00:00 +PREHOOK: query: explain +select * from flights_tiny_orc_partitioned_timestamp sort by fl_num, fl_time limit 25 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select * from flights_tiny_orc_partitioned_timestamp sort by fl_num, fl_time limit 25 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: flights_tiny_orc_partitioned_timestamp + Statistics: Num rows: 137 Data size: 33968 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), dest_city_name (type: string), fl_date (type: date), arr_delay (type: float), fl_num (type: int), fl_time (type: timestamp) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 137 Data size: 33968 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: int), _col5 (type: timestamp) + sort order: ++ + Statistics: Num rows: 137 Data size: 33968 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: date), _col3 (type: float) + Execution mode: vectorized + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: date), VALUE._col3 (type: float), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: timestamp) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 137 Data size: 33968 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 25 + Statistics: Num rows: 25 Data size: 6175 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: int), _col5 (type: timestamp) + sort order: ++ + Statistics: Num rows: 25 Data size: 6175 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: date), _col3 (type: float) + Execution mode: vectorized + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: date), VALUE._col3 (type: float), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: timestamp) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 25 Data size: 6175 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 25 + Statistics: Num rows: 25 Data size: 6175 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 25 Data size: 6175 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: 25 + Processor Tree: + ListSink + +PREHOOK: query: select * from flights_tiny_orc_partitioned_timestamp sort by fl_num, fl_time limit 25 +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 +#### A masked pattern was here #### +POSTHOOK: query: select * from flights_tiny_orc_partitioned_timestamp sort by fl_num, fl_time limit 25 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 +#### A masked pattern was here #### +Baltimore New York 2010-10-23 18.0 272 2010-10-23 07:00:00 +Baltimore New York 2010-10-30 14.0 272 2010-10-30 07:00:00 +Chicago New York 2010-10-20 42.0 361 2010-10-20 07:00:00 +Chicago New York 2010-10-21 142.0 361 2010-10-21 07:00:00 +Chicago New York 2010-10-22 2.0 361 2010-10-22 07:00:00 +Chicago New York 2010-10-24 10.0 361 2010-10-24 07:00:00 +Chicago New York 2010-10-25 31.0 361 2010-10-25 07:00:00 +Chicago New York 2010-10-26 12.0 361 2010-10-26 07:00:00 +Chicago New York 2010-10-27 148.0 361 2010-10-27 07:00:00 +Chicago New York 2010-10-28 2.0 361 2010-10-28 07:00:00 +Chicago New York 2010-10-29 -12.0 361 2010-10-29 07:00:00 +Chicago New York 2010-10-31 -25.0 361 2010-10-31 07:00:00 +Chicago New York 2010-10-23 3.0 384 2010-10-23 07:00:00 +Chicago New York 2010-10-30 -6.0 384 2010-10-30 07:00:00 +Chicago New York 2010-10-23 32.0 426 2010-10-23 07:00:00 +Chicago New York 2010-10-30 -10.0 426 2010-10-30 07:00:00 +Chicago New York 2010-10-23 1.0 650 2010-10-23 07:00:00 +Chicago New York 2010-10-30 -5.0 650 2010-10-30 07:00:00 +Chicago New York 2010-10-20 24.0 897 2010-10-20 07:00:00 +Chicago New York 2010-10-21 77.0 897 2010-10-21 07:00:00 +Chicago New York 2010-10-22 24.0 897 2010-10-22 07:00:00 +Chicago New York 2010-10-24 113.0 897 2010-10-24 07:00:00 +Chicago New York 2010-10-25 -1.0 897 2010-10-25 07:00:00 +Chicago New York 2010-10-26 0.0 897 2010-10-26 07:00:00 +Chicago New York 2010-10-27 -11.0 897 2010-10-27 07:00:00 +PREHOOK: query: explain +select fl_time, count(*) from flights_tiny_orc_partitioned_timestamp group by fl_time +PREHOOK: type: QUERY +POSTHOOK: query: explain +select fl_time, count(*) from flights_tiny_orc_partitioned_timestamp group by fl_time +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: flights_tiny_orc_partitioned_timestamp + Statistics: Num rows: 137 Data size: 33968 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: fl_time (type: timestamp) + outputColumnNames: fl_time + Statistics: Num rows: 137 Data size: 33968 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: fl_time (type: timestamp) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 137 Data size: 33968 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: timestamp) + sort order: + + Map-reduce partition columns: _col0 (type: timestamp) + Statistics: Num rows: 137 Data size: 33968 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: vectorized + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: timestamp) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 68 Data size: 16860 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: timestamp), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 68 Data size: 16860 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 68 Data size: 16860 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select fl_time, count(*) from flights_tiny_orc_partitioned_timestamp group by fl_time +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 +#### A masked pattern was here #### +POSTHOOK: query: select fl_time, count(*) from flights_tiny_orc_partitioned_timestamp group by fl_time +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 +#### A masked pattern was here #### +2010-10-20 07:00:00 11 +2010-10-21 07:00:00 12 +2010-10-22 07:00:00 11 +2010-10-23 07:00:00 12 +2010-10-24 07:00:00 12 +2010-10-25 07:00:00 12 +2010-10-26 07:00:00 13 +2010-10-27 07:00:00 11 +2010-10-28 07:00:00 12 +2010-10-29 07:00:00 12 +2010-10-30 07:00:00 11 +2010-10-31 07:00:00 8 diff --git a/ql/src/test/results/clientpositive/tez/vector_reduce_groupby_decimal.q.out b/ql/src/test/results/clientpositive/tez/vector_reduce_groupby_decimal.q.out new file mode 100644 index 0000000..14acfc5 --- /dev/null +++ b/ql/src/test/results/clientpositive/tez/vector_reduce_groupby_decimal.q.out @@ -0,0 +1,209 @@ +PREHOOK: query: CREATE TABLE decimal_test STORED AS ORC AS SELECT cint, cdouble, CAST (((cdouble*22.1)/37) AS DECIMAL(20,10)) AS cdecimal1, CAST (((cdouble*9.3)/13) AS DECIMAL(23,14)) AS cdecimal2 FROM alltypesorc +WHERE cint is not null and cdouble is not null +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: database:default +PREHOOK: Output: default@decimal_test +POSTHOOK: query: CREATE TABLE decimal_test STORED AS ORC AS SELECT cint, cdouble, CAST (((cdouble*22.1)/37) AS DECIMAL(20,10)) AS cdecimal1, CAST (((cdouble*9.3)/13) AS DECIMAL(23,14)) AS cdecimal2 FROM alltypesorc +WHERE cint is not null and cdouble is not null +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: database:default +POSTHOOK: Output: default@decimal_test +PREHOOK: query: EXPLAIN +SELECT cint, cdouble, cdecimal1, cdecimal2, min(cdecimal1) as min_decimal1 FROM decimal_test +WHERE cdecimal1 is not null and cdecimal2 is not null +GROUP BY cint, cdouble, cdecimal1, cdecimal2 +ORDER BY cint, cdouble, cdecimal1, cdecimal2 +LIMIT 50 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT cint, cdouble, cdecimal1, cdecimal2, min(cdecimal1) as min_decimal1 FROM decimal_test +WHERE cdecimal1 is not null and cdecimal2 is not null +GROUP BY cint, cdouble, cdecimal1, cdecimal2 +ORDER BY cint, cdouble, cdecimal1, cdecimal2 +LIMIT 50 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: decimal_test + Statistics: Num rows: 6102 Data size: 1440072 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (cdecimal1 is not null and cdecimal2 is not null) (type: boolean) + Statistics: Num rows: 1526 Data size: 360136 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cint (type: int), cdouble (type: double), cdecimal1 (type: decimal(20,10)), cdecimal2 (type: decimal(23,14)) + outputColumnNames: cint, cdouble, cdecimal1, cdecimal2 + Statistics: Num rows: 1526 Data size: 360136 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: min(cdecimal1) + keys: cint (type: int), cdouble (type: double), cdecimal1 (type: decimal(20,10)), cdecimal2 (type: decimal(23,14)) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1526 Data size: 360136 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: double), _col2 (type: decimal(20,10)), _col3 (type: decimal(23,14)) + sort order: ++++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: double), _col2 (type: decimal(20,10)), _col3 (type: decimal(23,14)) + Statistics: Num rows: 1526 Data size: 360136 Basic stats: COMPLETE Column stats: NONE + value expressions: _col4 (type: decimal(20,10)) + Execution mode: vectorized + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: min(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: double), KEY._col2 (type: decimal(20,10)), KEY._col3 (type: decimal(23,14)) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 763 Data size: 180068 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: double), _col2 (type: decimal(20,10)), _col3 (type: decimal(23,14)), _col4 (type: decimal(20,10)) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 763 Data size: 180068 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: double), _col2 (type: decimal(20,10)), _col3 (type: decimal(23,14)) + sort order: ++++ + Statistics: Num rows: 763 Data size: 180068 Basic stats: COMPLETE Column stats: NONE + value expressions: _col4 (type: decimal(20,10)) + Execution mode: vectorized + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: double), KEY.reducesinkkey2 (type: decimal(20,10)), KEY.reducesinkkey3 (type: decimal(23,14)), VALUE._col0 (type: decimal(20,10)) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 763 Data size: 180068 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 50 + Statistics: Num rows: 50 Data size: 11800 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 50 Data size: 11800 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: 50 + Processor Tree: + ListSink + +PREHOOK: query: SELECT cint, cdouble, cdecimal1, cdecimal2, min(cdecimal1) as min_decimal1 FROM decimal_test +WHERE cdecimal1 is not null and cdecimal2 is not null +GROUP BY cint, cdouble, cdecimal1, cdecimal2 +ORDER BY cint, cdouble, cdecimal1, cdecimal2 +LIMIT 50 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_test +#### A masked pattern was here #### +POSTHOOK: query: SELECT cint, cdouble, cdecimal1, cdecimal2, min(cdecimal1) as min_decimal1 FROM decimal_test +WHERE cdecimal1 is not null and cdecimal2 is not null +GROUP BY cint, cdouble, cdecimal1, cdecimal2 +ORDER BY cint, cdouble, cdecimal1, cdecimal2 +LIMIT 50 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_test +#### A masked pattern was here #### +-1073051226 -7382.0 -4409.2486486486 -5280.969230769231 -4409.2486486486 +-1072081801 8373.0 5001.1702702703 5989.915384615385 5001.1702702703 +-1072076362 -5470.0 -3267.2162162162 -3913.1538461538466 -3267.2162162162 +-1070883071 -741.0 -442.5972972973 -530.1 -442.5972972973 +-1070551679 -947.0 -565.6405405405 -677.4692307692308 -565.6405405405 +-1069512165 11417.0 6819.3432432432 8167.546153846154 6819.3432432432 +-1069109166 8390.0 5011.3243243243 6002.076923076923 5011.3243243243 +-1068623584 -14005.0 -8365.1486486486 -10018.961538461539 -8365.1486486486 +-1067386090 -3977.0 -2375.4513513514 -2845.084615384616 -2375.4513513514 +-1066922682 -9987.0 -5965.2081081081 -7144.546153846154 -5965.2081081081 +-1066226047 -9439.0 -5637.8891891892 -6752.515384615385 -5637.8891891892 +-1065117869 2538.0 1515.9405405405 1815.646153846154 1515.9405405405 +-1064949302 6454.0 3854.9567567568 4617.092307692308 3854.9567567568 +-1063498122 -11480.0 -6856.9729729730 -8212.615384615387 -6856.9729729730 +-1062973443 10541.0 6296.1108108108 7540.869230769231 6296.1108108108 +-1061614989 -4234.0 -2528.9567567568 -3028.938461538462 -2528.9567567568 +-1061057428 -1085.0 -648.0675675676 -776.1923076923077 -648.0675675676 +-1059941909 8782.0 5245.4648648649 6282.507692307693 5245.4648648649 +-1059338191 7322.0 4373.4108108108 5238.046153846154 4373.4108108108 +-1059047258 12452.0 7437.5459459459 8907.969230769231 7437.5459459459 +-1056684111 13991.0 8356.7864864865 10008.946153846155 8356.7864864865 +-1055945837 13690.0 8177.0 9793.615384615387 8177.0 +-1055669248 2570.0 1535.0540540541 1838.538461538462 1535.0540540541 +-1055316250 -14990.0 -8953.4864864865 -10723.615384615385 -8953.4864864865 +-1053385587 14504.0 8663.2 10375.938461538462 8663.2 +-1053238077 -3704.0 -2212.3891891892 -2649.784615384616 -2212.3891891892 +-1052745800 -12404.0 -7408.8756756757 -8873.630769230771 -7408.8756756757 +-1052322972 -7433.0 -4439.7108108108 -5317.453846153847 -4439.7108108108 +-1050684541 -8261.0 -4934.2729729730 -5909.792307692308 -4934.2729729730 +-1050657303 -6999.0 -4180.4837837838 -5006.976923076923 -4180.4837837838 +-1050165799 8634.0 5157.0648648649 6176.63076923077 5157.0648648649 +-1048934049 -524.0 -312.9837837838 -374.86153846153854 -312.9837837838 +-1046399794 4130.0 2466.8378378378 2954.5384615384614 2466.8378378378 +-1045867222 -8034.0 -4798.6864864865 -5747.400000000001 -4798.6864864865 +-1045196363 -5039.0 -3009.7810810811 -3604.823076923077 -3009.7810810811 +-1045181724 -5706.0 -3408.1783783784 -4081.9846153846156 -3408.1783783784 +-1045087657 -5865.0 -3503.1486486486 -4195.7307692307695 -3503.1486486486 +-1044207190 5381.0 3214.0567567568 3849.4846153846156 3214.0567567568 +-1044093617 -3422.0 -2043.9513513514 -2448.046153846154 -2043.9513513514 +-1043573508 16216.0 9685.7729729730 11600.676923076924 9685.7729729730 +-1043132597 12302.0 7347.9513513514 8800.66153846154 7347.9513513514 +-1043082182 9180.0 5483.1891891892 6567.2307692307695 5483.1891891892 +-1042805968 5133.0 3065.9270270270 3672.0692307692307 3065.9270270270 +-1042712895 9296.0 5552.4756756757 6650.215384615385 5552.4756756757 +-1042396242 9583.0 5723.9000000000 6855.53076923077 5723.9000000000 +-1041734429 -836.0 -499.3405405405 -598.0615384615385 -499.3405405405 +-1041391389 -12970.0 -7746.9459459459 -9278.538461538463 -7746.9459459459 +-1041252354 756.0 451.5567567568 540.8307692307692 451.5567567568 +-1039776293 13704.0 8185.3621621622 9803.630769230771 8185.3621621622 +-1039762548 -3802.0 -2270.9243243243 -2719.8923076923083 -2270.9243243243 +PREHOOK: query: SELECT sum(hash(*)) + FROM (SELECT cint, cdouble, cdecimal1, cdecimal2, min(cdecimal1) as min_decimal1 FROM decimal_test + WHERE cdecimal1 is not null and cdecimal2 is not null + GROUP BY cint, cdouble, cdecimal1, cdecimal2 + ORDER BY cint, cdouble, cdecimal1, cdecimal2 + LIMIT 50) as q +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_test +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(*)) + FROM (SELECT cint, cdouble, cdecimal1, cdecimal2, min(cdecimal1) as min_decimal1 FROM decimal_test + WHERE cdecimal1 is not null and cdecimal2 is not null + GROUP BY cint, cdouble, cdecimal1, cdecimal2 + ORDER BY cint, cdouble, cdecimal1, cdecimal2 + LIMIT 50) as q +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_test +#### A masked pattern was here #### +12703057972 +PREHOOK: query: SELECT sum(hash(*)) + FROM (SELECT cint, cdouble, cdecimal1, cdecimal2, min(cdecimal1) as min_decimal1 FROM decimal_test + WHERE cdecimal1 is not null and cdecimal2 is not null + GROUP BY cint, cdouble, cdecimal1, cdecimal2 + ORDER BY cint, cdouble, cdecimal1, cdecimal2 + LIMIT 50) as q +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_test +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(*)) + FROM (SELECT cint, cdouble, cdecimal1, cdecimal2, min(cdecimal1) as min_decimal1 FROM decimal_test + WHERE cdecimal1 is not null and cdecimal2 is not null + GROUP BY cint, cdouble, cdecimal1, cdecimal2 + ORDER BY cint, cdouble, cdecimal1, cdecimal2 + LIMIT 50) as q +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_test +#### A masked pattern was here #### +12703057972 diff --git a/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out b/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out index b0a5bcb..5cf41f0 100644 --- a/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out +++ b/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out @@ -1920,10 +1920,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (ds is not null and (ds = '2008-04-08')) (type: boolean) + filterExpr: (ds = '2008-04-08') (type: boolean) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: ds (type: string) + expressions: '2008-04-08' (type: string) outputColumnNames: ds Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -2595,7 +2595,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: ds (type: string) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - value expressions: hr (type: string) Map 5 Map Operator Tree: TableScan @@ -2660,14 +2659,12 @@ STAGE PLANS: condition map: Inner Join 0 to 1 condition expressions: - 0 {VALUE._col2} + 0 1 - outputColumnNames: _col3 Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col3 (type: string) + key expressions: '11' (type: string) sort order: + - Map-reduce partition columns: _col3 (type: string) Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE Reducer 3 Reduce Operator Tree: @@ -2758,15 +2755,15 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart_hour - filterExpr: (hr is not null and (hr = 13)) (type: boolean) + filterExpr: (hr = 13) (type: boolean) Statistics: Num rows: 2 Data size: 344 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (hr is not null and (hr = 13)) (type: boolean) - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + predicate: (hr = 13) (type: boolean) + Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: '13' (type: string) sort order: + - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: NONE Map 6 Map Operator Tree: TableScan @@ -2803,9 +2800,9 @@ STAGE PLANS: condition expressions: 0 1 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 1 Data size: 189 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 1 Data size: 189 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -4407,10 +4404,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (ds is not null and (ds = '2008-04-08')) (type: boolean) + filterExpr: (ds = '2008-04-08') (type: boolean) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: ds (type: string) + expressions: '2008-04-08' (type: string) outputColumnNames: ds Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -4803,12 +4800,11 @@ STAGE PLANS: condition map: Inner Join 0 to 1 condition expressions: - 0 {hr} + 0 1 keys: 0 ds (type: string) 1 ds (type: string) - outputColumnNames: _col3 input vertices: 1 Map 4 Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE @@ -4819,7 +4815,7 @@ STAGE PLANS: 0 1 keys: - 0 _col3 (type: string) + 0 '11' (type: string) 1 '11' (type: string) input vertices: 1 Map 3 @@ -4952,24 +4948,44 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) + Map 1 <- Map 4 (BROADCAST_EDGE) + Map 2 <- Map 1 (BROADCAST_EDGE) + Reducer 3 <- Map 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 - Map 3 + Map 2 Map Operator Tree: TableScan alias: srcpart_hour - filterExpr: (hr is not null and (hr = 13)) (type: boolean) + filterExpr: (hr = 13) (type: boolean) Statistics: Num rows: 2 Data size: 344 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (hr is not null and (hr = 13)) (type: boolean) - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Reduce Output Operator - key expressions: '13' (type: string) - sort order: + - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + predicate: (hr = 13) (type: boolean) + Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 + keys: + 0 '13' (type: string) + 1 '13' (type: string) + input vertices: + 0 Map 1 + Statistics: Num rows: 1 Data size: 189 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 1 Data size: 189 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) Map 4 Map Operator Tree: TableScan @@ -4985,7 +5001,7 @@ STAGE PLANS: Map-reduce partition columns: ds (type: string) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Execution mode: vectorized - Reducer 2 + Reducer 3 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) diff --git a/ql/src/test/results/clientpositive/timestamp_comparison2.q.out b/ql/src/test/results/clientpositive/timestamp_comparison2.q.out new file mode 100644 index 0000000..76ac21d --- /dev/null +++ b/ql/src/test/results/clientpositive/timestamp_comparison2.q.out @@ -0,0 +1,54 @@ +PREHOOK: query: -- Test timestamp-to-numeric comparison +select count(*) +FROM alltypesorc +WHERE +((ctinyint != 0) + AND + (((ctimestamp1 <= 0) + OR ((ctinyint = cint) OR (cstring2 LIKE 'ss'))) + AND ((988888 < cdouble) + OR ((ctimestamp2 > -29071) AND (3569 >= cdouble))))) +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +POSTHOOK: query: -- Test timestamp-to-numeric comparison +select count(*) +FROM alltypesorc +WHERE +((ctinyint != 0) + AND + (((ctimestamp1 <= 0) + OR ((ctinyint = cint) OR (cstring2 LIKE 'ss'))) + AND ((988888 < cdouble) + OR ((ctimestamp2 > -29071) AND (3569 >= cdouble))))) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +1826 +PREHOOK: query: -- Should have same result as previous query +select count(*) +FROM alltypesorc +WHERE +((ctinyint != 0) + AND + (((ctimestamp1 <= timestamp('1969-12-31 16:00:00')) + OR ((ctinyint = cint) OR (cstring2 LIKE 'ss'))) + AND ((988888 < cdouble) + OR ((ctimestamp2 > timestamp('1969-12-31 07:55:29')) AND (3569 >= cdouble))))) +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +POSTHOOK: query: -- Should have same result as previous query +select count(*) +FROM alltypesorc +WHERE +((ctinyint != 0) + AND + (((ctimestamp1 <= timestamp('1969-12-31 16:00:00')) + OR ((ctinyint = cint) OR (cstring2 LIKE 'ss'))) + AND ((988888 < cdouble) + OR ((ctimestamp2 > timestamp('1969-12-31 07:55:29')) AND (3569 >= cdouble))))) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +1826 diff --git a/ql/src/test/results/clientpositive/transform_acid.q.out b/ql/src/test/results/clientpositive/transform_acid.q.out new file mode 100644 index 0000000..704a261 --- /dev/null +++ b/ql/src/test/results/clientpositive/transform_acid.q.out @@ -0,0 +1,31 @@ +PREHOOK: query: -- EXCLUDE_OS_WINDOWS + +create table transform_acid(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@transform_acid +POSTHOOK: query: -- EXCLUDE_OS_WINDOWS + +create table transform_acid(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@transform_acid +PREHOOK: query: insert into table transform_acid select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint < 0 order by cint limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@transform_acid +POSTHOOK: query: insert into table transform_acid select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint < 0 order by cint limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@transform_acid +POSTHOOK: Lineage: transform_acid.a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: transform_acid.b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: SELECT transform(*) USING 'transform_acid_grep.sh' AS (col string) FROM transform_acid +PREHOOK: type: QUERY +PREHOOK: Input: default@transform_acid +#### A masked pattern was here #### +POSTHOOK: query: SELECT transform(*) USING 'transform_acid_grep.sh' AS (col string) FROM transform_acid +POSTHOOK: type: QUERY +POSTHOOK: Input: default@transform_acid +#### A masked pattern was here #### +a diff --git a/ql/src/test/results/clientpositive/transform_ppr2.q.out b/ql/src/test/results/clientpositive/transform_ppr2.q.out index acc6d0b..d09746e 100644 --- a/ql/src/test/results/clientpositive/transform_ppr2.q.out +++ b/ql/src/test/results/clientpositive/transform_ppr2.q.out @@ -106,7 +106,7 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE GatherStats: false Select Operator - expressions: ds (type: string), key (type: string), value (type: string) + expressions: '2008-04-08' (type: string), key (type: string), value (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Transform Operator diff --git a/ql/src/test/results/clientpositive/truncate_column_list_bucket.q.out b/ql/src/test/results/clientpositive/truncate_column_list_bucket.q.out index fc4a30b..8745b13 100644 --- a/ql/src/test/results/clientpositive/truncate_column_list_bucket.q.out +++ b/ql/src/test/results/clientpositive/truncate_column_list_bucket.q.out @@ -110,7 +110,7 @@ STAGE PLANS: predicate: (key = '484') (type: boolean) Statistics: Num rows: 8 Data size: 828 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: '484' (type: string), value (type: string), part (type: string) + expressions: '484' (type: string), value (type: string), '1' (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 8 Data size: 828 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -249,7 +249,7 @@ STAGE PLANS: predicate: (key = '0') (type: boolean) Statistics: Num rows: 8 Data size: 828 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: '0' (type: string), value (type: string), part (type: string) + expressions: '0' (type: string), value (type: string), '1' (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 8 Data size: 828 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/union15.q.out b/ql/src/test/results/clientpositive/union15.q.out index 9cac42b..0e940fd 100644 --- a/ql/src/test/results/clientpositive/union15.q.out +++ b/ql/src/test/results/clientpositive/union15.q.out @@ -73,12 +73,12 @@ STAGE PLANS: keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 2400 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2400 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: bigint) TableScan alias: s2 @@ -98,12 +98,12 @@ STAGE PLANS: keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 2400 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2400 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: bigint) TableScan alias: s3 @@ -123,12 +123,12 @@ STAGE PLANS: keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 2400 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2400 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -136,14 +136,14 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1200 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1200 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 12 Data size: 1200 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/union17.q.out b/ql/src/test/results/clientpositive/union17.q.out index 699fc7c..2c2aa4a 100644 --- a/ql/src/test/results/clientpositive/union17.q.out +++ b/ql/src/test/results/clientpositive/union17.q.out @@ -150,14 +150,14 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: final outputColumnNames: _col0, _col1 - Statistics: Num rows: 125 Data size: 24000 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 250 Data size: 48000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 125 Data size: 24000 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 250 Data size: 48000 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 125 Data size: 24000 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 250 Data size: 48000 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -193,14 +193,14 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: final outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 125 Data size: 47000 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 250 Data size: 94000 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 125 Data size: 47000 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 250 Data size: 94000 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 125 Data size: 47000 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 250 Data size: 94000 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/union19.q.out b/ql/src/test/results/clientpositive/union19.q.out index 3e8fea3..1d83ab4 100644 --- a/ql/src/test/results/clientpositive/union19.q.out +++ b/ql/src/test/results/clientpositive/union19.q.out @@ -90,12 +90,12 @@ STAGE PLANS: keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 24000 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 24000 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: bigint) Select Operator expressions: _col0 (type: string), _col1 (type: string), _col1 (type: string) @@ -127,12 +127,12 @@ STAGE PLANS: keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 24000 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 24000 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: bigint) Select Operator expressions: _col0 (type: string), _col1 (type: string), _col1 (type: string) @@ -152,14 +152,14 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 125 Data size: 12500 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 125 Data size: 12500 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 125 Data size: 12500 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/union20.q.out b/ql/src/test/results/clientpositive/union20.q.out index d66c7a6..1aa82e5 100644 --- a/ql/src/test/results/clientpositive/union20.q.out +++ b/ql/src/test/results/clientpositive/union20.q.out @@ -72,57 +72,57 @@ STAGE PLANS: Map Operator Tree: TableScan Union - Statistics: Num rows: 84 Data size: 1153 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 167 Data size: 2035 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 84 Data size: 1153 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 167 Data size: 2035 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string) TableScan alias: s4 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 10) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 84 Data size: 1153 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 167 Data size: 2035 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 84 Data size: 1153 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 167 Data size: 2035 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string) TableScan alias: s2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 10) and key is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 84 Data size: 1153 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 167 Data size: 2035 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 84 Data size: 1153 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 167 Data size: 2035 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string) TableScan Union - Statistics: Num rows: 84 Data size: 1153 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 167 Data size: 2035 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 84 Data size: 1153 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 167 Data size: 2035 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator @@ -132,14 +132,14 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} {VALUE._col0} 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 7056 Data size: 1919232 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 27889 Data size: 7585808 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 7056 Data size: 1919232 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 27889 Data size: 7585808 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 7056 Data size: 1919232 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 27889 Data size: 7585808 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/union21.q.out b/ql/src/test/results/clientpositive/union21.q.out index fb2328d..a754e05 100644 --- a/ql/src/test/results/clientpositive/union21.q.out +++ b/ql/src/test/results/clientpositive/union21.q.out @@ -60,12 +60,12 @@ STAGE PLANS: keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 761 Data size: 70773 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 761 Data size: 70773 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: bigint) TableScan alias: src_thrift @@ -85,12 +85,12 @@ STAGE PLANS: keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 761 Data size: 70773 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 761 Data size: 70773 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: bigint) TableScan alias: src @@ -110,12 +110,12 @@ STAGE PLANS: keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 761 Data size: 70773 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 761 Data size: 70773 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: bigint) TableScan alias: src @@ -135,12 +135,12 @@ STAGE PLANS: keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 761 Data size: 70773 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 761 Data size: 70773 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: bigint) TableScan alias: src @@ -160,12 +160,12 @@ STAGE PLANS: keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 761 Data size: 70773 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 761 Data size: 70773 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -173,14 +173,14 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 380 Data size: 38000 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 380 Data size: 38000 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 380 Data size: 38000 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/union24.q.out b/ql/src/test/results/clientpositive/union24.q.out index 77ae018..1decd24 100644 --- a/ql/src/test/results/clientpositive/union24.q.out +++ b/ql/src/test/results/clientpositive/union24.q.out @@ -807,13 +807,13 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: (key is not null and (key < 10)) (type: boolean) - Statistics: Num rows: 51 Data size: 244 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 51 Data size: 244 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: count (type: bigint) auto parallelism: false @@ -823,13 +823,13 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: (key is not null and (key < 10)) (type: boolean) - Statistics: Num rows: 51 Data size: 244 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 51 Data size: 244 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE tag: 0 auto parallelism: false Path -> Alias: @@ -935,11 +935,11 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 {VALUE._col0} outputColumnNames: _col0, _col6 - Statistics: Num rows: 56 Data size: 268 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 113 Data size: 543 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col6 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 56 Data size: 268 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 113 Data size: 543 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 @@ -964,17 +964,17 @@ STAGE PLANS: TableScan GatherStats: false Union - Statistics: Num rows: 262 Data size: 1256 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 319 Data size: 1531 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 262 Data size: 1256 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 319 Data size: 1531 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 262 Data size: 1256 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 319 Data size: 1531 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -1003,17 +1003,17 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 262 Data size: 1256 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 319 Data size: 1531 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 262 Data size: 1256 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 319 Data size: 1531 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 262 Data size: 1256 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 319 Data size: 1531 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -1042,17 +1042,17 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 262 Data size: 1256 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 319 Data size: 1531 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 262 Data size: 1256 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 319 Data size: 1531 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 262 Data size: 1256 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 319 Data size: 1531 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -1383,13 +1383,13 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: (key is not null and (key < 10)) (type: boolean) - Statistics: Num rows: 51 Data size: 244 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 51 Data size: 244 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE tag: 1 auto parallelism: false TableScan @@ -1398,13 +1398,13 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: (key is not null and (key < 10)) (type: boolean) - Statistics: Num rows: 51 Data size: 244 Basic stats: COMPLETE Column stats: NONE + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: key (type: string) sort order: + Map-reduce partition columns: key (type: string) - Statistics: Num rows: 51 Data size: 244 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE tag: 0 auto parallelism: false Path -> Alias: @@ -1510,17 +1510,17 @@ STAGE PLANS: 0 {KEY.reducesinkkey0} 1 outputColumnNames: _col0 - Statistics: Num rows: 56 Data size: 268 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 113 Data size: 543 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 56 Data size: 268 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 113 Data size: 543 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 56 Data size: 268 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 113 Data size: 543 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 @@ -1548,7 +1548,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 56 Data size: 268 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 113 Data size: 543 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: bigint) auto parallelism: false @@ -1584,11 +1584,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 28 Data size: 134 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 56 Data size: 269 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 28 Data size: 134 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 56 Data size: 269 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 @@ -1613,17 +1613,17 @@ STAGE PLANS: TableScan GatherStats: false Union - Statistics: Num rows: 234 Data size: 1122 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 262 Data size: 1257 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 234 Data size: 1122 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 262 Data size: 1257 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 234 Data size: 1122 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 262 Data size: 1257 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -1652,17 +1652,17 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 234 Data size: 1122 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 262 Data size: 1257 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 234 Data size: 1122 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 262 Data size: 1257 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 234 Data size: 1122 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 262 Data size: 1257 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -1691,17 +1691,17 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 234 Data size: 1122 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 262 Data size: 1257 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 234 Data size: 1122 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 262 Data size: 1257 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 234 Data size: 1122 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 262 Data size: 1257 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git a/ql/src/test/results/clientpositive/union26.q.out b/ql/src/test/results/clientpositive/union26.q.out index 4f9e0d4..cdc558a 100644 --- a/ql/src/test/results/clientpositive/union26.q.out +++ b/ql/src/test/results/clientpositive/union26.q.out @@ -103,88 +103,91 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Lateral View Forward - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Lateral View Join Operator - outputColumnNames: _col0, _col1, _col7 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Union - Statistics: Num rows: 1275 Data size: 13545 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1275 Data size: 13545 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1275 Data size: 13545 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1275 Data size: 13545 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) - Select Operator - expressions: array(1,2,3) (type: array) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - UDTF Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - function name: explode + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((ds = '2008-04-08') and (hr = '11')) (type: boolean) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Lateral View Forward + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Lateral View Join Operator outputColumnNames: _col0, _col1, _col7 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4000 Data size: 42496 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4000 Data size: 42496 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 1275 Data size: 13545 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4275 Data size: 45417 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1275 Data size: 13545 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4275 Data size: 45417 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1275 Data size: 13545 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4275 Data size: 45417 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1275 Data size: 13545 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4275 Data size: 45417 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) + Select Operator + expressions: array(1,2,3) (type: array) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + UDTF Operator + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + function name: explode + Lateral View Join Operator + outputColumnNames: _col0, _col1, _col7 + Statistics: Num rows: 4000 Data size: 42496 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4000 Data size: 42496 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 4275 Data size: 45417 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4275 Data size: 45417 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(1) + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4275 Data size: 45417 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 4275 Data size: 45417 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) TableScan Union - Statistics: Num rows: 1275 Data size: 13545 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4275 Data size: 45417 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1275 Data size: 13545 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4275 Data size: 45417 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1275 Data size: 13545 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4275 Data size: 45417 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1275 Data size: 13545 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4275 Data size: 45417 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Reduce Operator Tree: Group By Operator @@ -192,14 +195,14 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 637 Data size: 6767 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2137 Data size: 22703 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: bigint), _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 637 Data size: 6767 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2137 Data size: 22703 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 637 Data size: 6767 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2137 Data size: 22703 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -235,6 +238,8 @@ PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### POSTHOOK: query: SELECT count(1) as counts, @@ -260,6 +265,8 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### 10 100 val_100 10 103 val_103 @@ -594,6 +601,8 @@ PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### POSTHOOK: query: SELECT count(1) as counts, @@ -619,6 +628,8 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### 10 100 val_100 10 103 val_103 @@ -953,6 +964,8 @@ PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### POSTHOOK: query: SELECT count(1) as counts, @@ -978,6 +991,8 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### 10 100 val_100 10 103 val_103 diff --git a/ql/src/test/results/clientpositive/union27.q.out b/ql/src/test/results/clientpositive/union27.q.out index da37707..036bf8a 100644 --- a/ql/src/test/results/clientpositive/union27.q.out +++ b/ql/src/test/results/clientpositive/union27.q.out @@ -46,45 +46,45 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key = 97) and key is not null) (type: boolean) - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 97) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: '97' (type: string) sort order: + - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE TableScan alias: jackson_sev_add Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key = 97) and key is not null) (type: boolean) - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 97) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: string) outputColumnNames: _col1 - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: '97' (type: string) sort order: + - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan alias: dim_pho Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key = 97) and key is not null) (type: boolean) - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + predicate: (key = 97) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: string) outputColumnNames: _col1 - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: '97' (type: string) sort order: + - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator @@ -94,14 +94,14 @@ STAGE PLANS: 0 1 {VALUE._col0} outputColumnNames: _col6 - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: '97' (type: string), _col6 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/union29.q.out b/ql/src/test/results/clientpositive/union29.q.out index 41dde4e..3a60d0a 100644 --- a/ql/src/test/results/clientpositive/union29.q.out +++ b/ql/src/test/results/clientpositive/union29.q.out @@ -75,25 +75,19 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: string) + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Union + Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.union_subq_union + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE @@ -102,25 +96,19 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: string) + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Union + Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.union_subq_union + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union Stage: Stage-7 Conditional Operator diff --git a/ql/src/test/results/clientpositive/union30.q.out b/ql/src/test/results/clientpositive/union30.q.out index b506536..59934c6 100644 --- a/ql/src/test/results/clientpositive/union30.q.out +++ b/ql/src/test/results/clientpositive/union30.q.out @@ -48,17 +48,16 @@ select key, value from src POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1, Stage-11 + Stage-2 depends on stages: Stage-1, Stage-10 Stage-3 depends on stages: Stage-2 - Stage-4 depends on stages: Stage-3 - Stage-10 depends on stages: Stage-4 , consists of Stage-7, Stage-6, Stage-8 - Stage-7 - Stage-0 depends on stages: Stage-7, Stage-6, Stage-9 - Stage-5 depends on stages: Stage-0 + Stage-9 depends on stages: Stage-3 , consists of Stage-6, Stage-5, Stage-7 Stage-6 - Stage-8 - Stage-9 depends on stages: Stage-8 - Stage-11 is a root stage + Stage-0 depends on stages: Stage-6, Stage-5, Stage-8 + Stage-4 depends on stages: Stage-0 + Stage-5 + Stage-7 + Stage-8 depends on stages: Stage-7 + Stage-10 is a root stage STAGE PLANS: Stage: Stage-1 @@ -135,42 +134,6 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Union - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - TableScan - Union - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan Union Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column stats: NONE Select Operator @@ -206,11 +169,32 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_subq_union + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.union_subq_union - Stage: Stage-10 + Stage: Stage-9 Conditional Operator - Stage: Stage-7 + Stage: Stage-6 Move Operator files: hdfs directory: true @@ -226,10 +210,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_subq_union - Stage: Stage-5 + Stage: Stage-4 Stats-Aggr Operator - Stage: Stage-6 + Stage: Stage-5 Map Reduce Map Operator Tree: TableScan @@ -241,7 +225,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_subq_union - Stage: Stage-8 + Stage: Stage-7 Map Reduce Map Operator Tree: TableScan @@ -253,13 +237,13 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.union_subq_union - Stage: Stage-9 + Stage: Stage-8 Move Operator files: hdfs directory: true #### A masked pattern was here #### - Stage: Stage-11 + Stage: Stage-10 Map Reduce Map Operator Tree: TableScan diff --git a/ql/src/test/results/clientpositive/union34.q.out b/ql/src/test/results/clientpositive/union34.q.out index dc46ac7..0ab57d6 100644 --- a/ql/src/test/results/clientpositive/union34.q.out +++ b/ql/src/test/results/clientpositive/union34.q.out @@ -121,22 +121,16 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE - Union + Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + value expressions: _col1 (type: string) TableScan alias: src10_4 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE @@ -145,22 +139,16 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE - Union + Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + value expressions: _col1 (type: string) TableScan alias: src10_2 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE @@ -375,22 +363,16 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE - Union + Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + value expressions: _col1 (type: string) TableScan alias: src10_4 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE @@ -399,22 +381,16 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 20 Data size: 208 Basic stats: COMPLETE Column stats: NONE - Union + Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Statistics: Num rows: 25 Data size: 265 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + value expressions: _col1 (type: string) Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) diff --git a/ql/src/test/results/clientpositive/union7.q.out b/ql/src/test/results/clientpositive/union7.q.out index 46ed3db..727ff57 100644 --- a/ql/src/test/results/clientpositive/union7.q.out +++ b/ql/src/test/results/clientpositive/union7.q.out @@ -69,12 +69,12 @@ STAGE PLANS: keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: bigint) TableScan alias: s2 @@ -94,12 +94,12 @@ STAGE PLANS: keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -107,14 +107,14 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 600 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 600 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 600 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/union_date_trim.q.out b/ql/src/test/results/clientpositive/union_date_trim.q.out new file mode 100644 index 0000000..e0682e6 --- /dev/null +++ b/ql/src/test/results/clientpositive/union_date_trim.q.out @@ -0,0 +1,54 @@ +PREHOOK: query: drop table if exists testDate +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists testDate +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table testDate(id int, dt date) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@testDate +POSTHOOK: query: create table testDate(id int, dt date) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@testDate +PREHOOK: query: insert into table testDate select 1, '2014-04-07' from src where key=100 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@testdate +POSTHOOK: query: insert into table testDate select 1, '2014-04-07' from src where key=100 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@testdate +POSTHOOK: Lineage: testdate.dt EXPRESSION [] +POSTHOOK: Lineage: testdate.id SIMPLE [] +PREHOOK: query: insert into table testDate select 2, '2014-04-08' from src where key=100 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@testdate +POSTHOOK: query: insert into table testDate select 2, '2014-04-08' from src where key=100 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@testdate +POSTHOOK: Lineage: testdate.dt EXPRESSION [] +POSTHOOK: Lineage: testdate.id SIMPLE [] +PREHOOK: query: insert into table testDate select 3, '2014-04-09' from src where key=100 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@testdate +POSTHOOK: query: insert into table testDate select 3, '2014-04-09' from src where key=100 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@testdate +POSTHOOK: Lineage: testdate.dt EXPRESSION [] +POSTHOOK: Lineage: testdate.id SIMPLE [] +PREHOOK: query: --- without the fix following query will throw HiveException: Incompatible types for union operator +insert into table testDate select id, tm from (select id, dt as tm from testDate where id = 1 union all select id, dt as tm from testDate where id = 2 union all select id, trim(Cast (dt as string)) as tm from testDate where id = 3 ) a +PREHOOK: type: QUERY +PREHOOK: Input: default@testdate +PREHOOK: Output: default@testdate +POSTHOOK: query: --- without the fix following query will throw HiveException: Incompatible types for union operator +insert into table testDate select id, tm from (select id, dt as tm from testDate where id = 1 union all select id, dt as tm from testDate where id = 2 union all select id, trim(Cast (dt as string)) as tm from testDate where id = 3 ) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testdate +POSTHOOK: Output: default@testdate +POSTHOOK: Lineage: testdate.dt EXPRESSION [] +POSTHOOK: Lineage: testdate.id EXPRESSION [(testdate)testdate.FieldSchema(name:id, type:int, comment:null), (testdate)testdate.FieldSchema(name:id, type:int, comment:null), (testdate)testdate.FieldSchema(name:id, type:int, comment:null), ] diff --git a/ql/src/test/results/clientpositive/union_ppr.q.out b/ql/src/test/results/clientpositive/union_ppr.q.out index 399dfdf..992cbea 100644 --- a/ql/src/test/results/clientpositive/union_ppr.q.out +++ b/ql/src/test/results/clientpositive/union_ppr.q.out @@ -121,17 +121,17 @@ STAGE PLANS: predicate: (key < 100) (type: boolean) Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + expressions: key (type: string), value (type: string), hr (type: string) + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE Union Statistics: Num rows: 666 Data size: 7074 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string) + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 666 Data size: 7074 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + key expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), _col3 (type: string) sort order: ++++ Statistics: Num rows: 666 Data size: 7074 Basic stats: COMPLETE Column stats: NONE tag: -1 @@ -145,17 +145,17 @@ STAGE PLANS: predicate: (key < 100) (type: boolean) Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + expressions: key (type: string), value (type: string), hr (type: string) + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE Union Statistics: Num rows: 666 Data size: 7074 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string) + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 666 Data size: 7074 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + key expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), _col3 (type: string) sort order: ++++ Statistics: Num rows: 666 Data size: 7074 Basic stats: COMPLETE Column stats: NONE tag: -1 @@ -261,7 +261,7 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string) + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), '2008-04-08' (type: string), KEY.reducesinkkey3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 666 Data size: 7074 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/union_remove_10.q.out b/ql/src/test/results/clientpositive/union_remove_10.q.out index b078793..344ed9c 100644 --- a/ql/src/test/results/clientpositive/union_remove_10.q.out +++ b/ql/src/test/results/clientpositive/union_remove_10.q.out @@ -78,13 +78,13 @@ select * FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-7 depends on stages: Stage-2, Stage-8 , consists of Stage-4, Stage-3, Stage-5 - Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-6 depends on stages: Stage-1, Stage-7, Stage-8 , consists of Stage-3, Stage-2, Stage-4 Stage-3 - Stage-5 - Stage-6 depends on stages: Stage-5 + Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 + Stage-2 + Stage-4 + Stage-5 depends on stages: Stage-4 + Stage-7 is a root stage Stage-8 is a root stage STAGE PLANS: @@ -123,55 +123,17 @@ STAGE PLANS: Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1 - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Union - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1 - TableScan - alias: inputtbl1 - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: key (type: string), UDFToLong(2) (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Union - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1 - - Stage: Stage-7 + Stage: Stage-6 Conditional Operator - Stage: Stage-4 + Stage: Stage-3 Move Operator files: hdfs directory: true @@ -187,27 +149,27 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Stage: Stage-3 + Stage: Stage-2 Merge File Operator Map Operator Tree: RCFile Merge Operator merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-5 + Stage: Stage-4 Merge File Operator Map Operator Tree: RCFile Merge Operator merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-6 + Stage: Stage-5 Move Operator files: hdfs directory: true #### A masked pattern was here #### - Stage: Stage-8 + Stage: Stage-7 Map Reduce Map Operator Tree: TableScan @@ -226,6 +188,25 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 + Stage: Stage-8 + Map Reduce + Map Operator Tree: + TableScan + alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: key (type: string), UDFToLong(2) (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1 + PREHOOK: query: insert overwrite table outputTbl1 SELECT * FROM ( @@ -255,7 +236,6 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@inputtbl1 POSTHOOK: Output: default@outputtbl1 POSTHOOK: Lineage: outputtbl1.key EXPRESSION [(inputtbl1)inputtbl1.FieldSchema(name:key, type:string, comment:null), (inputtbl1)inputtbl1.FieldSchema(name:key, type:string, comment:null), (inputtbl1)inputtbl1.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: outputtbl1.values EXPRESSION [(inputtbl1)inputtbl1.null, ] PREHOOK: query: desc formatted outputTbl1 PREHOOK: type: DESCTABLE PREHOOK: Input: default@outputtbl1 diff --git a/ql/src/test/results/clientpositive/union_remove_11.q.out b/ql/src/test/results/clientpositive/union_remove_11.q.out index 82d91a9..6db85b6 100644 --- a/ql/src/test/results/clientpositive/union_remove_11.q.out +++ b/ql/src/test/results/clientpositive/union_remove_11.q.out @@ -97,25 +97,19 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Union - Statistics: Num rows: 0 Data size: 60 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: int) + expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 0 Data size: 60 Basic stats: PARTIAL Column stats: NONE - Union + Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1 + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1 TableScan alias: inputtbl1 Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE @@ -145,25 +139,19 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Union - Statistics: Num rows: 0 Data size: 60 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: int) + expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 0 Data size: 60 Basic stats: PARTIAL Column stats: NONE - Union + Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1 + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1 Stage: Stage-6 Conditional Operator diff --git a/ql/src/test/results/clientpositive/union_remove_19.q.out b/ql/src/test/results/clientpositive/union_remove_19.q.out index fb4c192..a030c24 100644 --- a/ql/src/test/results/clientpositive/union_remove_19.q.out +++ b/ql/src/test/results/clientpositive/union_remove_19.q.out @@ -299,7 +299,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 7 Data size: 7 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: bigint) + expressions: '7' (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 Statistics: Num rows: 7 Data size: 7 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -354,7 +354,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 7 Data size: 7 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: bigint) + expressions: '7' (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 Statistics: Num rows: 7 Data size: 7 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/union_remove_25.q.out b/ql/src/test/results/clientpositive/union_remove_25.q.out index 3b20e15..28940f7 100644 --- a/ql/src/test/results/clientpositive/union_remove_25.q.out +++ b/ql/src/test/results/clientpositive/union_remove_25.q.out @@ -292,8 +292,8 @@ STAGE PLANS: alias: srcpart Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string), ds (type: string) - outputColumnNames: _col0, _col1, _col2 + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 500 @@ -301,17 +301,17 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 500 Data size: 5000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + value expressions: _col0 (type: string), _col1 (type: string) Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string) - outputColumnNames: _col0, _col1, _col2 + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5000 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 500 Statistics: Num rows: 500 Data size: 5000 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), _col2 (type: string) + expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), '2008-04-08' (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5000 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -342,8 +342,8 @@ STAGE PLANS: alias: srcpart Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string), ds (type: string) - outputColumnNames: _col0, _col1, _col2 + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 500 @@ -351,17 +351,17 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 500 Data size: 5000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + value expressions: _col0 (type: string), _col1 (type: string) Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string) - outputColumnNames: _col0, _col1, _col2 + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5000 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 500 Statistics: Num rows: 500 Data size: 5000 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), _col2 (type: string) + expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), '2008-04-08' (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5000 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -476,8 +476,8 @@ STAGE PLANS: alias: srcpart Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + expressions: key (type: string), value (type: string), hr (type: string) + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 1000 @@ -485,17 +485,17 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1000 Data size: 10000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string) Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col3 (type: string) + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 1000 Data size: 10000 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 1000 Statistics: Num rows: 1000 Data size: 10000 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), _col2 (type: string), _col3 (type: string) + expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), '2008-04-08' (type: string), _col3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1000 Data size: 10000 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -527,8 +527,8 @@ STAGE PLANS: alias: srcpart Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + expressions: key (type: string), value (type: string), hr (type: string) + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 1000 @@ -536,17 +536,17 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1000 Data size: 10000 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string) Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col3 (type: string) + outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 1000 Data size: 10000 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 1000 Statistics: Num rows: 1000 Data size: 10000 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), _col2 (type: string), _col3 (type: string) + expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), '2008-04-08' (type: string), _col3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1000 Data size: 10000 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/union_view.q.out b/ql/src/test/results/clientpositive/union_view.q.out index bbbf5e7..ce987c3 100644 --- a/ql/src/test/results/clientpositive/union_view.q.out +++ b/ql/src/test/results/clientpositive/union_view.q.out @@ -54,7 +54,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__src_union_1_src_union_1_key_idx__ + alias: default.default__src_union_1_src_union_1_key_idx__ filterExpr: ((key = 86) and (ds = '1')) (type: boolean) Filter Operator predicate: (key = 86) (type: boolean) @@ -85,7 +85,7 @@ STAGE PLANS: predicate: (key = 86) (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 86 (type: int), value (type: string), ds (type: string) + expressions: 86 (type: int), value (type: string), '1' (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -113,7 +113,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__src_union_2_src_union_2_key_idx__ + alias: default.default__src_union_2_src_union_2_key_idx__ filterExpr: ((key = 86) and (ds = '2')) (type: boolean) Filter Operator predicate: (key = 86) (type: boolean) @@ -144,7 +144,7 @@ STAGE PLANS: predicate: (key = 86) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 86 (type: int), value (type: string), ds (type: string) + expressions: 86 (type: int), value (type: string), '2' (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -172,7 +172,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__src_union_3_src_union_3_key_idx__ + alias: default.default__src_union_3_src_union_3_key_idx__ filterExpr: ((key = 86) and (ds = '3')) (type: boolean) Filter Operator predicate: (key = 86) (type: boolean) @@ -203,7 +203,7 @@ STAGE PLANS: predicate: (key = 86) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 86 (type: int), value (type: string), ds (type: string) + expressions: 86 (type: int), value (type: string), '3' (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -236,7 +236,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__src_union_1_src_union_1_key_idx__ + alias: default.default__src_union_1_src_union_1_key_idx__ filterExpr: (ds = '1') (type: boolean) Select Operator expressions: _bucketname (type: string), _offsets (type: array) @@ -307,7 +307,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__src_union_2_src_union_2_key_idx__ + alias: default.default__src_union_2_src_union_2_key_idx__ filterExpr: (ds = '2') (type: boolean) Select Operator expressions: _bucketname (type: string), _offsets (type: array) @@ -378,7 +378,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__src_union_3_src_union_3_key_idx__ + alias: default.default__src_union_3_src_union_3_key_idx__ filterExpr: (ds = '3') (type: boolean) Select Operator expressions: _bucketname (type: string), _offsets (type: array) @@ -452,7 +452,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__src_union_1_src_union_1_key_idx__ + alias: default.default__src_union_1_src_union_1_key_idx__ filterExpr: ((key = 86) and (ds = '1')) (type: boolean) Filter Operator predicate: (key = 86) (type: boolean) @@ -483,13 +483,13 @@ STAGE PLANS: predicate: (key = 86) (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: value (type: string), ds (type: string) - outputColumnNames: _col1, _col2 + expressions: value (type: string) + outputColumnNames: _col1 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Union Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 86 (type: int), _col1 (type: string), _col2 (type: string) + expressions: 86 (type: int), _col1 (type: string), '1' (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -517,7 +517,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__src_union_2_src_union_2_key_idx__ + alias: default.default__src_union_2_src_union_2_key_idx__ filterExpr: ((key = 86) and (ds = '2')) (type: boolean) Filter Operator predicate: (key = 86) (type: boolean) @@ -548,13 +548,13 @@ STAGE PLANS: predicate: (key = 86) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: value (type: string), ds (type: string) - outputColumnNames: _col1, _col2 + expressions: value (type: string) + outputColumnNames: _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Union Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 86 (type: int), _col1 (type: string), _col2 (type: string) + expressions: 86 (type: int), _col1 (type: string), '2' (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -582,7 +582,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__src_union_3_src_union_3_key_idx__ + alias: default.default__src_union_3_src_union_3_key_idx__ filterExpr: ((key = 86) and (ds = '3')) (type: boolean) Filter Operator predicate: (key = 86) (type: boolean) @@ -613,13 +613,13 @@ STAGE PLANS: predicate: (key = 86) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: value (type: string), ds (type: string) - outputColumnNames: _col1, _col2 + expressions: value (type: string) + outputColumnNames: _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Union Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 86 (type: int), _col1 (type: string), _col2 (type: string) + expressions: 86 (type: int), _col1 (type: string), '3' (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -651,7 +651,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__src_union_2_src_union_2_key_idx__ + alias: default.default__src_union_2_src_union_2_key_idx__ filterExpr: (key = 86) (type: boolean) Filter Operator predicate: (key = 86) (type: boolean) @@ -757,7 +757,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__src_union_1_src_union_1_key_idx__ + alias: default.default__src_union_1_src_union_1_key_idx__ filterExpr: (key = 86) (type: boolean) Filter Operator predicate: (key = 86) (type: boolean) @@ -781,7 +781,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__src_union_3_src_union_3_key_idx__ + alias: default.default__src_union_3_src_union_3_key_idx__ filterExpr: (key = 86) (type: boolean) Filter Operator predicate: (key = 86) (type: boolean) @@ -828,7 +828,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__src_union_1_src_union_1_key_idx__ + alias: default.default__src_union_1_src_union_1_key_idx__ filterExpr: (ds = '1') (type: boolean) Select Operator expressions: _bucketname (type: string), _offsets (type: array) @@ -903,7 +903,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__src_union_2_src_union_2_key_idx__ + alias: default.default__src_union_2_src_union_2_key_idx__ filterExpr: (ds = '2') (type: boolean) Select Operator expressions: _bucketname (type: string), _offsets (type: array) @@ -978,7 +978,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__src_union_3_src_union_3_key_idx__ + alias: default.default__src_union_3_src_union_3_key_idx__ filterExpr: (ds = '3') (type: boolean) Select Operator expressions: _bucketname (type: string), _offsets (type: array) @@ -1056,7 +1056,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__src_union_3_src_union_3_key_idx__ + alias: default.default__src_union_3_src_union_3_key_idx__ filterExpr: ((key = 86) and (ds = '4')) (type: boolean) Filter Operator predicate: (key = 86) (type: boolean) @@ -1087,13 +1087,13 @@ STAGE PLANS: predicate: (key = 86) (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: value (type: string), ds (type: string) - outputColumnNames: _col1, _col2 + expressions: value (type: string) + outputColumnNames: _col1 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Union Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 86 (type: int), _col1 (type: string), _col2 (type: string) + expressions: 86 (type: int), _col1 (type: string), '4' (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -1122,7 +1122,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: default__src_union_3_src_union_3_key_idx__ + alias: default.default__src_union_3_src_union_3_key_idx__ filterExpr: (ds = '4') (type: boolean) Select Operator expressions: _bucketname (type: string), _offsets (type: array) diff --git a/ql/src/test/results/clientpositive/update_all_partitioned.q.out b/ql/src/test/results/clientpositive/update_all_partitioned.q.out index 533dd88..286e2ed 100644 --- a/ql/src/test/results/clientpositive/update_all_partitioned.q.out +++ b/ql/src/test/results/clientpositive/update_all_partitioned.q.out @@ -85,21 +85,15 @@ POSTHOOK: Input: default@acid_uap@ds=today POSTHOOK: Input: default@acid_uap@ds=tomorrow #### A masked pattern was here #### -1073279343 fred today --1073279343 oj1YrV5Wa today -1073051226 fred today -1072910839 fred today --1072081801 dPkN74F7 today -1072081801 fred today -1072076362 fred today -1071480828 fred today --1071363017 Anj0oF today -1071363017 fred today --1070883071 0ruyd6Y50JpdGRf6HqD today -1070883071 fred today -1070551679 fred today --1070551679 iUR3Q today -1069736047 fred today --1069736047 k17Am8uPHWk02cEf1jet today 762 fred tomorrow 762 fred tomorrow 762 fred tomorrow diff --git a/ql/src/test/results/clientpositive/vector_bucket.q.out b/ql/src/test/results/clientpositive/vector_bucket.q.out new file mode 100644 index 0000000..952c033 --- /dev/null +++ b/ql/src/test/results/clientpositive/vector_bucket.q.out @@ -0,0 +1,94 @@ +PREHOOK: query: CREATE TABLE non_orc_table(a INT, b STRING) CLUSTERED BY(a) INTO 2 BUCKETS STORED AS sequencefile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@non_orc_table +POSTHOOK: query: CREATE TABLE non_orc_table(a INT, b STRING) CLUSTERED BY(a) INTO 2 BUCKETS STORED AS sequencefile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@non_orc_table +PREHOOK: query: explain +insert into table non_orc_table values(1, 'one'),(1, 'one'), (2, 'two'),(3, 'three') +PREHOOK: type: QUERY +POSTHOOK: query: explain +insert into table non_orc_table values(1, 'one'),(1, 'one'), (2, 'two'),(3, 'three') +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: values__tmp__table__1 + Statistics: Num rows: 0 Data size: 26 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: tmp_values_col1 (type: string), tmp_values_col2 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 26 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + sort order: + Map-reduce partition columns: UDFToInteger(_col0) (type: int) + Statistics: Num rows: 0 Data size: 26 Basic stats: PARTIAL Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + Reduce Operator Tree: + Extract + Statistics: Num rows: 0 Data size: 26 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 26 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 26 Basic stats: PARTIAL Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.non_orc_table + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.non_orc_table + + Stage: Stage-2 + Stats-Aggr Operator + +PREHOOK: query: select a, b from non_orc_table order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@non_orc_table +#### A masked pattern was here #### +POSTHOOK: query: select a, b from non_orc_table order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@non_orc_table +#### A masked pattern was here #### +PREHOOK: query: insert into table non_orc_table values(1, 'one'),(1, 'one'), (2, 'two'),(3, 'three') +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__2 +PREHOOK: Output: default@non_orc_table +POSTHOOK: query: insert into table non_orc_table values(1, 'one'),(1, 'one'), (2, 'two'),(3, 'three') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__2 +POSTHOOK: Output: default@non_orc_table +POSTHOOK: Lineage: non_orc_table.a EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: non_orc_table.b SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: select a, b from non_orc_table order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@non_orc_table +#### A masked pattern was here #### +POSTHOOK: query: select a, b from non_orc_table order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@non_orc_table +#### A masked pattern was here #### +1 one +1 one +2 two +3 three diff --git a/ql/src/test/results/clientpositive/vector_data_types.q.out b/ql/src/test/results/clientpositive/vector_data_types.q.out index 6b0598c..07d58ed 100644 --- a/ql/src/test/results/clientpositive/vector_data_types.q.out +++ b/ql/src/test/results/clientpositive/vector_data_types.q.out @@ -171,6 +171,17 @@ NULL 409 65536 4294967490 46.97 25.92 false fred miller 2013-03-01 09:11:58.7031 120 331 65539 4294967324 88.02 40.94 true holly nixon 2013-03-01 09:11:58.703262 96.64 yard duty 71 298 65540 4294967510 91.63 9.6 false gabriella zipper 2013-03-01 09:11:58.70314 68.14 geology 114 385 65541 4294967458 73.48 34.97 true oscar quirinius 2013-03-01 09:11:58.703143 72.33 xylophone band +PREHOOK: query: SELECT SUM(HASH(*)) +FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i) as q +PREHOOK: type: QUERY +PREHOOK: Input: default@over1korc +#### A masked pattern was here #### +POSTHOOK: query: SELECT SUM(HASH(*)) +FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i) as q +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1korc +#### A masked pattern was here #### +-17045922556 PREHOOK: query: EXPLAIN select t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i LIMIT 20 PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN select t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i LIMIT 20 @@ -226,23 +237,34 @@ POSTHOOK: query: SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc OR POSTHOOK: type: QUERY POSTHOOK: Input: default@over1korc #### A masked pattern was here #### -108 301 65536 4294967357 90.05 17.59 true ethan johnson 1860-11-12 20:05:55.011470936 75.70 undecided -118 497 65536 4294967381 50.32 12.72 false david nixon 1860-11-12 20:05:55.011484936 83.48 values clariffication -18 280 65536 4294967320 32.92 45.94 false holly white 1860-11-12 20:05:55.011285936 58.86 topology -69 489 65536 4294967404 33.52 17.99 false oscar ichabod 1860-11-12 20:05:55.011446936 32.68 topology -27 405 65536 4294967508 82.24 29.41 true oscar ovid 1860-11-12 20:05:55.011365936 16.85 biology -42 495 65536 4294967431 43.57 46.81 false tom johnson 1860-11-12 20:05:55.011444936 62.25 american history -NULL 409 65536 4294967490 46.97 25.92 false fred miller 1860-11-12 20:05:55.011315936 33.45 history -59 431 65537 4294967326 11.34 2.9 true oscar robinson 1860-11-12 20:05:55.011360936 81.04 zync studies -63 458 65537 4294967463 21.94 49.71 true fred van buren 1860-11-12 20:05:55.011477936 99.34 wind surfing -121 355 65537 4294967437 85.9 10.99 true calvin brown 1860-11-12 20:05:55.011453936 4.49 yard duty -72 267 65537 4294967460 55.2 42.89 false oscar carson 1860-11-12 20:05:55.011352936 51.91 topology -22 264 65537 4294967419 8.07 10.71 false david xylophone 1860-11-12 20:05:55.011335936 11.87 undecided -101 283 65538 4294967527 84.03 27.29 true yuri brown 1860-11-12 20:05:55.011505936 44.24 biology -109 376 65538 4294967453 59.61 35.62 true priscilla xylophone 1860-11-12 20:05:55.011485936 9.53 study skills -83 353 65538 4294967339 0.08 23.91 true holly thompson 1860-11-12 20:05:55.011356936 30.27 quiet hour -29 331 65539 4294967420 73.18 28.96 true ethan brown 1860-11-12 20:05:55.011293936 58.85 zync studies -70 430 65539 4294967534 18.89 43.84 true tom carson 1860-11-12 20:05:55.011381936 21.93 joggying -120 331 65539 4294967324 88.02 40.94 true holly nixon 1860-11-12 20:05:55.011461936 96.64 yard duty -71 298 65540 4294967510 91.63 9.6 false gabriella zipper 1860-11-12 20:05:55.011339936 68.14 geology -114 385 65541 4294967458 73.48 34.97 true oscar quirinius 1860-11-12 20:05:55.011342936 72.33 xylophone band +108 301 65536 4294967357 90.05 17.59 true ethan johnson 2013-03-01 09:11:58.703271 75.70 undecided +118 497 65536 4294967381 50.32 12.72 false david nixon 2013-03-01 09:11:58.703285 83.48 values clariffication +18 280 65536 4294967320 32.92 45.94 false holly white 2013-03-01 09:11:58.703086 58.86 topology +69 489 65536 4294967404 33.52 17.99 false oscar ichabod 2013-03-01 09:11:58.703247 32.68 topology +27 405 65536 4294967508 82.24 29.41 true oscar ovid 2013-03-01 09:11:58.703166 16.85 biology +42 495 65536 4294967431 43.57 46.81 false tom johnson 2013-03-01 09:11:58.703245 62.25 american history +NULL 409 65536 4294967490 46.97 25.92 false fred miller 2013-03-01 09:11:58.703116 33.45 history +59 431 65537 4294967326 11.34 2.9 true oscar robinson 2013-03-01 09:11:58.703161 81.04 zync studies +63 458 65537 4294967463 21.94 49.71 true fred van buren 2013-03-01 09:11:58.703278 99.34 wind surfing +121 355 65537 4294967437 85.9 10.99 true calvin brown 2013-03-01 09:11:58.703254 4.49 yard duty +72 267 65537 4294967460 55.2 42.89 false oscar carson 2013-03-01 09:11:58.703153 51.91 topology +22 264 65537 4294967419 8.07 10.71 false david xylophone 2013-03-01 09:11:58.703136 11.87 undecided +101 283 65538 4294967527 84.03 27.29 true yuri brown 2013-03-01 09:11:58.703306 44.24 biology +109 376 65538 4294967453 59.61 35.62 true priscilla xylophone 2013-03-01 09:11:58.703286 9.53 study skills +83 353 65538 4294967339 0.08 23.91 true holly thompson 2013-03-01 09:11:58.703157 30.27 quiet hour +29 331 65539 4294967420 73.18 28.96 true ethan brown 2013-03-01 09:11:58.703094 58.85 zync studies +70 430 65539 4294967534 18.89 43.84 true tom carson 2013-03-01 09:11:58.703182 21.93 joggying +120 331 65539 4294967324 88.02 40.94 true holly nixon 2013-03-01 09:11:58.703262 96.64 yard duty +71 298 65540 4294967510 91.63 9.6 false gabriella zipper 2013-03-01 09:11:58.70314 68.14 geology +114 385 65541 4294967458 73.48 34.97 true oscar quirinius 2013-03-01 09:11:58.703143 72.33 xylophone band +PREHOOK: query: SELECT SUM(HASH(*)) +FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i) as q +PREHOOK: type: QUERY +PREHOOK: Input: default@over1korc +#### A masked pattern was here #### +POSTHOOK: query: SELECT SUM(HASH(*)) +FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i) as q +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1korc +#### A masked pattern was here #### +-17045922556 diff --git a/ql/src/test/results/clientpositive/vector_decimal_mapjoin.q.out b/ql/src/test/results/clientpositive/vector_decimal_mapjoin.q.out index cd68b85..193e6f7 100644 --- a/ql/src/test/results/clientpositive/vector_decimal_mapjoin.q.out +++ b/ql/src/test/results/clientpositive/vector_decimal_mapjoin.q.out @@ -48,8 +48,8 @@ STAGE PLANS: alias: l Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (cint is not null and (cint = 6981)) (type: boolean) - Statistics: Num rows: 3072 Data size: 541265 Basic stats: COMPLETE Column stats: NONE + predicate: (cint = 6981) (type: boolean) + Statistics: Num rows: 6144 Data size: 1082530 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: 0 {cdecimal1} @@ -65,8 +65,8 @@ STAGE PLANS: alias: r Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (cint is not null and (cint = 6981)) (type: boolean) - Statistics: Num rows: 3072 Data size: 541265 Basic stats: COMPLETE Column stats: NONE + predicate: (cint = 6981) (type: boolean) + Statistics: Num rows: 6144 Data size: 1082530 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -77,14 +77,14 @@ STAGE PLANS: 0 6981 (type: int) 1 6981 (type: int) outputColumnNames: _col1, _col9 - Statistics: Num rows: 3379 Data size: 595391 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6758 Data size: 1190783 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 6981 (type: int), 6981 (type: int), _col1 (type: decimal(20,10)), _col9 (type: decimal(23,14)) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 3379 Data size: 595391 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6758 Data size: 1190783 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3379 Data size: 595391 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6758 Data size: 1190783 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git a/ql/src/test/results/clientpositive/vector_groupby_reduce.q.out b/ql/src/test/results/clientpositive/vector_groupby_reduce.q.out new file mode 100644 index 0000000..438d6b6 --- /dev/null +++ b/ql/src/test/results/clientpositive/vector_groupby_reduce.q.out @@ -0,0 +1,453 @@ +PREHOOK: query: create table store_sales_txt +( + ss_sold_date_sk int, + ss_sold_time_sk int, + ss_item_sk int, + ss_customer_sk int, + ss_cdemo_sk int, + ss_hdemo_sk int, + ss_addr_sk int, + ss_store_sk int, + ss_promo_sk int, + ss_ticket_number int, + ss_quantity int, + ss_wholesale_cost float, + ss_list_price float, + ss_sales_price float, + ss_ext_discount_amt float, + ss_ext_sales_price float, + ss_ext_wholesale_cost float, + ss_ext_list_price float, + ss_ext_tax float, + ss_coupon_amt float, + ss_net_paid float, + ss_net_paid_inc_tax float, + ss_net_profit float +) +row format delimited fields terminated by '|' +stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@store_sales_txt +POSTHOOK: query: create table store_sales_txt +( + ss_sold_date_sk int, + ss_sold_time_sk int, + ss_item_sk int, + ss_customer_sk int, + ss_cdemo_sk int, + ss_hdemo_sk int, + ss_addr_sk int, + ss_store_sk int, + ss_promo_sk int, + ss_ticket_number int, + ss_quantity int, + ss_wholesale_cost float, + ss_list_price float, + ss_sales_price float, + ss_ext_discount_amt float, + ss_ext_sales_price float, + ss_ext_wholesale_cost float, + ss_ext_list_price float, + ss_ext_tax float, + ss_coupon_amt float, + ss_net_paid float, + ss_net_paid_inc_tax float, + ss_net_profit float +) +row format delimited fields terminated by '|' +stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@store_sales_txt +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/store_sales.txt' OVERWRITE INTO TABLE store_sales_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@store_sales_txt +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/store_sales.txt' OVERWRITE INTO TABLE store_sales_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@store_sales_txt +PREHOOK: query: create table store_sales +( + ss_sold_date_sk int, + ss_sold_time_sk int, + ss_item_sk int, + ss_customer_sk int, + ss_cdemo_sk int, + ss_hdemo_sk int, + ss_addr_sk int, + ss_store_sk int, + ss_promo_sk int, + ss_ticket_number int, + ss_quantity int, + ss_wholesale_cost float, + ss_list_price float, + ss_sales_price float, + ss_ext_discount_amt float, + ss_ext_sales_price float, + ss_ext_wholesale_cost float, + ss_ext_list_price float, + ss_ext_tax float, + ss_coupon_amt float, + ss_net_paid float, + ss_net_paid_inc_tax float, + ss_net_profit float +) +stored as orc +tblproperties ("orc.stripe.size"="33554432", "orc.compress.size"="16384") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@store_sales +POSTHOOK: query: create table store_sales +( + ss_sold_date_sk int, + ss_sold_time_sk int, + ss_item_sk int, + ss_customer_sk int, + ss_cdemo_sk int, + ss_hdemo_sk int, + ss_addr_sk int, + ss_store_sk int, + ss_promo_sk int, + ss_ticket_number int, + ss_quantity int, + ss_wholesale_cost float, + ss_list_price float, + ss_sales_price float, + ss_ext_discount_amt float, + ss_ext_sales_price float, + ss_ext_wholesale_cost float, + ss_ext_list_price float, + ss_ext_tax float, + ss_coupon_amt float, + ss_net_paid float, + ss_net_paid_inc_tax float, + ss_net_profit float +) +stored as orc +tblproperties ("orc.stripe.size"="33554432", "orc.compress.size"="16384") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@store_sales +PREHOOK: query: insert overwrite table store_sales +select +ss_sold_date_sk , + ss_sold_time_sk , + ss_item_sk , + ss_customer_sk , + ss_cdemo_sk , + ss_hdemo_sk , + ss_addr_sk , + ss_store_sk , + ss_promo_sk , + ss_ticket_number , + ss_quantity , + ss_wholesale_cost , + ss_list_price , + ss_sales_price , + ss_ext_discount_amt , + ss_ext_sales_price , + ss_ext_wholesale_cost , + ss_ext_list_price , + ss_ext_tax , + ss_coupon_amt , + ss_net_paid , + ss_net_paid_inc_tax , + ss_net_profit + from store_sales_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@store_sales_txt +PREHOOK: Output: default@store_sales +POSTHOOK: query: insert overwrite table store_sales +select +ss_sold_date_sk , + ss_sold_time_sk , + ss_item_sk , + ss_customer_sk , + ss_cdemo_sk , + ss_hdemo_sk , + ss_addr_sk , + ss_store_sk , + ss_promo_sk , + ss_ticket_number , + ss_quantity , + ss_wholesale_cost , + ss_list_price , + ss_sales_price , + ss_ext_discount_amt , + ss_ext_sales_price , + ss_ext_wholesale_cost , + ss_ext_list_price , + ss_ext_tax , + ss_coupon_amt , + ss_net_paid , + ss_net_paid_inc_tax , + ss_net_profit + from store_sales_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@store_sales_txt +POSTHOOK: Output: default@store_sales +POSTHOOK: Lineage: store_sales.ss_addr_sk SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_addr_sk, type:int, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_cdemo_sk SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_cdemo_sk, type:int, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_coupon_amt SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_coupon_amt, type:float, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_customer_sk SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_customer_sk, type:int, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_ext_discount_amt SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_ext_discount_amt, type:float, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_ext_list_price SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_ext_list_price, type:float, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_ext_sales_price SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_ext_sales_price, type:float, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_ext_tax SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_ext_tax, type:float, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_ext_wholesale_cost SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_ext_wholesale_cost, type:float, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_hdemo_sk SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_hdemo_sk, type:int, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_item_sk SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_item_sk, type:int, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_list_price SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_list_price, type:float, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_net_paid SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_net_paid, type:float, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_net_paid_inc_tax SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_net_paid_inc_tax, type:float, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_net_profit SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_net_profit, type:float, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_promo_sk SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_promo_sk, type:int, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_quantity SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_quantity, type:int, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_sales_price SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_sales_price, type:float, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_sold_date_sk SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_sold_date_sk, type:int, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_sold_time_sk SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_sold_time_sk, type:int, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_store_sk SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_store_sk, type:int, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_ticket_number SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_ticket_number, type:int, comment:null), ] +POSTHOOK: Lineage: store_sales.ss_wholesale_cost SIMPLE [(store_sales_txt)store_sales_txt.FieldSchema(name:ss_wholesale_cost, type:float, comment:null), ] +PREHOOK: query: explain +select + ss_ticket_number +from + store_sales +group by ss_ticket_number +limit 20 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select + ss_ticket_number +from + store_sales +group by ss_ticket_number +limit 20 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: store_sales + Statistics: Num rows: 1000 Data size: 88276 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ss_ticket_number (type: int) + outputColumnNames: ss_ticket_number + Statistics: Num rows: 1000 Data size: 88276 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: ss_ticket_number (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1000 Data size: 88276 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1000 Data size: 88276 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 44138 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 44138 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 20 + Statistics: Num rows: 20 Data size: 1760 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 20 Data size: 1760 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: 20 + Processor Tree: + ListSink + +PREHOOK: query: select + ss_ticket_number +from + store_sales +group by ss_ticket_number +limit 20 +PREHOOK: type: QUERY +PREHOOK: Input: default@store_sales +#### A masked pattern was here #### +POSTHOOK: query: select + ss_ticket_number +from + store_sales +group by ss_ticket_number +limit 20 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@store_sales +#### A masked pattern was here #### +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +PREHOOK: query: explain +select + min(ss_ticket_number) +from + (select + ss_ticket_number + from + store_sales + group by ss_ticket_number) a +group by ss_ticket_number +limit 20 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select + min(ss_ticket_number) +from + (select + ss_ticket_number + from + store_sales + group by ss_ticket_number) a +group by ss_ticket_number +limit 20 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: store_sales + Statistics: Num rows: 1000 Data size: 88276 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ss_ticket_number (type: int) + outputColumnNames: ss_ticket_number + Statistics: Num rows: 1000 Data size: 88276 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: ss_ticket_number (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1000 Data size: 88276 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1000 Data size: 88276 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 44138 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 44138 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: min(_col0) + keys: _col0 (type: int) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 22069 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 22069 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 20 + Statistics: Num rows: 20 Data size: 1760 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 20 Data size: 1760 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: 20 + Processor Tree: + ListSink + +PREHOOK: query: select + min(ss_ticket_number) +from + (select + ss_ticket_number + from + store_sales + group by ss_ticket_number) a +group by ss_ticket_number +limit 20 +PREHOOK: type: QUERY +PREHOOK: Input: default@store_sales +#### A masked pattern was here #### +POSTHOOK: query: select + min(ss_ticket_number) +from + (select + ss_ticket_number + from + store_sales + group by ss_ticket_number) a +group by ss_ticket_number +limit 20 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@store_sales +#### A masked pattern was here #### +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 diff --git a/ql/src/test/results/clientpositive/vector_partitioned_date_time.q.out b/ql/src/test/results/clientpositive/vector_partitioned_date_time.q.out new file mode 100644 index 0000000..f3eff07 --- /dev/null +++ b/ql/src/test/results/clientpositive/vector_partitioned_date_time.q.out @@ -0,0 +1,2042 @@ +PREHOOK: query: -- Check if vectorization code is handling partitioning on DATE and the other data types. + + +CREATE TABLE flights_tiny ( + origin_city_name STRING, + dest_city_name STRING, + fl_date DATE, + arr_delay FLOAT, + fl_num INT +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@flights_tiny +POSTHOOK: query: -- Check if vectorization code is handling partitioning on DATE and the other data types. + + +CREATE TABLE flights_tiny ( + origin_city_name STRING, + dest_city_name STRING, + fl_date DATE, + arr_delay FLOAT, + fl_num INT +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@flights_tiny +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/flights_tiny.txt.1' OVERWRITE INTO TABLE flights_tiny +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@flights_tiny +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/flights_tiny.txt.1' OVERWRITE INTO TABLE flights_tiny +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@flights_tiny +PREHOOK: query: CREATE TABLE flights_tiny_orc STORED AS ORC AS +SELECT origin_city_name, dest_city_name, fl_date, to_utc_timestamp(fl_date, 'America/Los_Angeles') as fl_time, arr_delay, fl_num +FROM flights_tiny +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@flights_tiny +PREHOOK: Output: database:default +PREHOOK: Output: default@flights_tiny_orc +POSTHOOK: query: CREATE TABLE flights_tiny_orc STORED AS ORC AS +SELECT origin_city_name, dest_city_name, fl_date, to_utc_timestamp(fl_date, 'America/Los_Angeles') as fl_time, arr_delay, fl_num +FROM flights_tiny +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@flights_tiny +POSTHOOK: Output: database:default +POSTHOOK: Output: default@flights_tiny_orc +PREHOOK: query: SELECT * FROM flights_tiny_orc +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM flights_tiny_orc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc +#### A masked pattern was here #### +Baltimore New York 2010-10-20 2010-10-20 07:00:00 -30.0 1064 +Baltimore New York 2010-10-20 2010-10-20 07:00:00 23.0 1142 +Baltimore New York 2010-10-20 2010-10-20 07:00:00 6.0 1599 +Chicago New York 2010-10-20 2010-10-20 07:00:00 42.0 361 +Chicago New York 2010-10-20 2010-10-20 07:00:00 24.0 897 +Chicago New York 2010-10-20 2010-10-20 07:00:00 15.0 1531 +Chicago New York 2010-10-20 2010-10-20 07:00:00 -6.0 1610 +Chicago New York 2010-10-20 2010-10-20 07:00:00 -2.0 3198 +Baltimore New York 2010-10-21 2010-10-21 07:00:00 17.0 1064 +Baltimore New York 2010-10-21 2010-10-21 07:00:00 105.0 1142 +Baltimore New York 2010-10-21 2010-10-21 07:00:00 28.0 1599 +Chicago New York 2010-10-21 2010-10-21 07:00:00 142.0 361 +Chicago New York 2010-10-21 2010-10-21 07:00:00 77.0 897 +Chicago New York 2010-10-21 2010-10-21 07:00:00 53.0 1531 +Chicago New York 2010-10-21 2010-10-21 07:00:00 -5.0 1610 +Chicago New York 2010-10-21 2010-10-21 07:00:00 51.0 3198 +Baltimore New York 2010-10-22 2010-10-22 07:00:00 -12.0 1064 +Baltimore New York 2010-10-22 2010-10-22 07:00:00 54.0 1142 +Baltimore New York 2010-10-22 2010-10-22 07:00:00 18.0 1599 +Chicago New York 2010-10-22 2010-10-22 07:00:00 2.0 361 +Chicago New York 2010-10-22 2010-10-22 07:00:00 24.0 897 +Chicago New York 2010-10-22 2010-10-22 07:00:00 16.0 1531 +Chicago New York 2010-10-22 2010-10-22 07:00:00 -6.0 1610 +Chicago New York 2010-10-22 2010-10-22 07:00:00 -11.0 3198 +Baltimore New York 2010-10-23 2010-10-23 07:00:00 18.0 272 +Baltimore New York 2010-10-23 2010-10-23 07:00:00 -10.0 1805 +Baltimore New York 2010-10-23 2010-10-23 07:00:00 6.0 3171 +Chicago New York 2010-10-23 2010-10-23 07:00:00 3.0 384 +Chicago New York 2010-10-23 2010-10-23 07:00:00 32.0 426 +Chicago New York 2010-10-23 2010-10-23 07:00:00 1.0 650 +Chicago New York 2010-10-23 2010-10-23 07:00:00 11.0 3085 +Baltimore New York 2010-10-24 2010-10-24 07:00:00 12.0 1599 +Baltimore New York 2010-10-24 2010-10-24 07:00:00 20.0 2571 +Chicago New York 2010-10-24 2010-10-24 07:00:00 10.0 361 +Chicago New York 2010-10-24 2010-10-24 07:00:00 113.0 897 +Chicago New York 2010-10-24 2010-10-24 07:00:00 -5.0 1531 +Chicago New York 2010-10-24 2010-10-24 07:00:00 -17.0 1610 +Chicago New York 2010-10-24 2010-10-24 07:00:00 -3.0 3198 +Baltimore New York 2010-10-25 2010-10-25 07:00:00 -25.0 1064 +Baltimore New York 2010-10-25 2010-10-25 07:00:00 92.0 1142 +Baltimore New York 2010-10-25 2010-10-25 07:00:00 106.0 1599 +Chicago New York 2010-10-25 2010-10-25 07:00:00 31.0 361 +Chicago New York 2010-10-25 2010-10-25 07:00:00 -1.0 897 +Chicago New York 2010-10-25 2010-10-25 07:00:00 43.0 1531 +Chicago New York 2010-10-25 2010-10-25 07:00:00 6.0 1610 +Chicago New York 2010-10-25 2010-10-25 07:00:00 -16.0 3198 +Baltimore New York 2010-10-26 2010-10-26 07:00:00 -22.0 1064 +Baltimore New York 2010-10-26 2010-10-26 07:00:00 123.0 1142 +Baltimore New York 2010-10-26 2010-10-26 07:00:00 90.0 1599 +Chicago New York 2010-10-26 2010-10-26 07:00:00 12.0 361 +Chicago New York 2010-10-26 2010-10-26 07:00:00 0.0 897 +Chicago New York 2010-10-26 2010-10-26 07:00:00 29.0 1531 +Chicago New York 2010-10-26 2010-10-26 07:00:00 -17.0 1610 +Chicago New York 2010-10-26 2010-10-26 07:00:00 6.0 3198 +Baltimore New York 2010-10-27 2010-10-27 07:00:00 -18.0 1064 +Baltimore New York 2010-10-27 2010-10-27 07:00:00 49.0 1142 +Baltimore New York 2010-10-27 2010-10-27 07:00:00 92.0 1599 +Chicago New York 2010-10-27 2010-10-27 07:00:00 148.0 361 +Chicago New York 2010-10-27 2010-10-27 07:00:00 -11.0 897 +Chicago New York 2010-10-27 2010-10-27 07:00:00 70.0 1531 +Chicago New York 2010-10-27 2010-10-27 07:00:00 8.0 1610 +Chicago New York 2010-10-27 2010-10-27 07:00:00 21.0 3198 +Baltimore New York 2010-10-28 2010-10-28 07:00:00 -4.0 1064 +Baltimore New York 2010-10-28 2010-10-28 07:00:00 -14.0 1142 +Baltimore New York 2010-10-28 2010-10-28 07:00:00 -14.0 1599 +Chicago New York 2010-10-28 2010-10-28 07:00:00 2.0 361 +Chicago New York 2010-10-28 2010-10-28 07:00:00 2.0 897 +Chicago New York 2010-10-28 2010-10-28 07:00:00 -11.0 1531 +Chicago New York 2010-10-28 2010-10-28 07:00:00 3.0 1610 +Chicago New York 2010-10-28 2010-10-28 07:00:00 -18.0 3198 +Baltimore New York 2010-10-29 2010-10-29 07:00:00 -24.0 1064 +Baltimore New York 2010-10-29 2010-10-29 07:00:00 21.0 1142 +Baltimore New York 2010-10-29 2010-10-29 07:00:00 -2.0 1599 +Chicago New York 2010-10-29 2010-10-29 07:00:00 -12.0 361 +Chicago New York 2010-10-29 2010-10-29 07:00:00 -11.0 897 +Chicago New York 2010-10-29 2010-10-29 07:00:00 15.0 1531 +Chicago New York 2010-10-29 2010-10-29 07:00:00 -18.0 1610 +Chicago New York 2010-10-29 2010-10-29 07:00:00 -4.0 3198 +Baltimore New York 2010-10-30 2010-10-30 07:00:00 14.0 272 +Baltimore New York 2010-10-30 2010-10-30 07:00:00 -1.0 1805 +Baltimore New York 2010-10-30 2010-10-30 07:00:00 5.0 3171 +Chicago New York 2010-10-30 2010-10-30 07:00:00 -6.0 384 +Chicago New York 2010-10-30 2010-10-30 07:00:00 -10.0 426 +Chicago New York 2010-10-30 2010-10-30 07:00:00 -5.0 650 +Chicago New York 2010-10-30 2010-10-30 07:00:00 -5.0 3085 +Baltimore New York 2010-10-31 2010-10-31 07:00:00 -1.0 1599 +Baltimore New York 2010-10-31 2010-10-31 07:00:00 -14.0 2571 +Chicago New York 2010-10-31 2010-10-31 07:00:00 -25.0 361 +Chicago New York 2010-10-31 2010-10-31 07:00:00 -18.0 897 +Chicago New York 2010-10-31 2010-10-31 07:00:00 -4.0 1531 +Chicago New York 2010-10-31 2010-10-31 07:00:00 -22.0 1610 +Chicago New York 2010-10-31 2010-10-31 07:00:00 -15.0 3198 +Cleveland New York 2010-10-30 2010-10-30 07:00:00 -23.0 2018 +Cleveland New York 2010-10-30 2010-10-30 07:00:00 -12.0 2932 +Cleveland New York 2010-10-29 2010-10-29 07:00:00 -4.0 2630 +Cleveland New York 2010-10-29 2010-10-29 07:00:00 -19.0 2646 +Cleveland New York 2010-10-29 2010-10-29 07:00:00 -12.0 3014 +Cleveland New York 2010-10-28 2010-10-28 07:00:00 3.0 2630 +Cleveland New York 2010-10-28 2010-10-28 07:00:00 -6.0 2646 +Cleveland New York 2010-10-28 2010-10-28 07:00:00 1.0 3014 +Cleveland New York 2010-10-27 2010-10-27 07:00:00 16.0 2630 +Cleveland New York 2010-10-27 2010-10-27 07:00:00 27.0 3014 +Cleveland New York 2010-10-26 2010-10-26 07:00:00 4.0 2630 +Cleveland New York 2010-10-26 2010-10-26 07:00:00 -27.0 2646 +Cleveland New York 2010-10-26 2010-10-26 07:00:00 -11.0 2662 +Cleveland New York 2010-10-26 2010-10-26 07:00:00 13.0 3014 +Cleveland New York 2010-10-25 2010-10-25 07:00:00 -4.0 2630 +Cleveland New York 2010-10-25 2010-10-25 07:00:00 81.0 2646 +Cleveland New York 2010-10-25 2010-10-25 07:00:00 42.0 3014 +Cleveland New York 2010-10-24 2010-10-24 07:00:00 5.0 2254 +Cleveland New York 2010-10-24 2010-10-24 07:00:00 -11.0 2630 +Cleveland New York 2010-10-24 2010-10-24 07:00:00 -20.0 2646 +Cleveland New York 2010-10-24 2010-10-24 07:00:00 -9.0 3014 +Cleveland New York 2010-10-23 2010-10-23 07:00:00 -21.0 2932 +Cleveland New York 2010-10-22 2010-10-22 07:00:00 1.0 2630 +Cleveland New York 2010-10-22 2010-10-22 07:00:00 -25.0 2646 +Cleveland New York 2010-10-22 2010-10-22 07:00:00 -3.0 3014 +Cleveland New York 2010-10-21 2010-10-21 07:00:00 3.0 2630 +Cleveland New York 2010-10-21 2010-10-21 07:00:00 29.0 2646 +Cleveland New York 2010-10-21 2010-10-21 07:00:00 72.0 3014 +Cleveland New York 2010-10-20 2010-10-20 07:00:00 -8.0 2630 +Cleveland New York 2010-10-20 2010-10-20 07:00:00 -15.0 3014 +Washington New York 2010-10-23 2010-10-23 07:00:00 -25.0 5832 +Washington New York 2010-10-23 2010-10-23 07:00:00 -21.0 5904 +Washington New York 2010-10-23 2010-10-23 07:00:00 -18.0 5917 +Washington New York 2010-10-30 2010-10-30 07:00:00 -27.0 5904 +Washington New York 2010-10-30 2010-10-30 07:00:00 -16.0 5917 +Washington New York 2010-10-20 2010-10-20 07:00:00 -2.0 7291 +Washington New York 2010-10-21 2010-10-21 07:00:00 22.0 7291 +Washington New York 2010-10-23 2010-10-23 07:00:00 -16.0 7274 +Washington New York 2010-10-24 2010-10-24 07:00:00 -26.0 7282 +Washington New York 2010-10-25 2010-10-25 07:00:00 9.0 7291 +Washington New York 2010-10-26 2010-10-26 07:00:00 4.0 7291 +Washington New York 2010-10-27 2010-10-27 07:00:00 26.0 7291 +Washington New York 2010-10-28 2010-10-28 07:00:00 45.0 7291 +Washington New York 2010-10-29 2010-10-29 07:00:00 1.0 7291 +Washington New York 2010-10-31 2010-10-31 07:00:00 -18.0 7282 +PREHOOK: query: select * from flights_tiny_orc sort by fl_num, fl_date limit 25 +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc +#### A masked pattern was here #### +POSTHOOK: query: select * from flights_tiny_orc sort by fl_num, fl_date limit 25 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc +#### A masked pattern was here #### +Baltimore New York 2010-10-23 2010-10-23 07:00:00 18.0 272 +Baltimore New York 2010-10-30 2010-10-30 07:00:00 14.0 272 +Chicago New York 2010-10-20 2010-10-20 07:00:00 42.0 361 +Chicago New York 2010-10-21 2010-10-21 07:00:00 142.0 361 +Chicago New York 2010-10-22 2010-10-22 07:00:00 2.0 361 +Chicago New York 2010-10-24 2010-10-24 07:00:00 10.0 361 +Chicago New York 2010-10-25 2010-10-25 07:00:00 31.0 361 +Chicago New York 2010-10-26 2010-10-26 07:00:00 12.0 361 +Chicago New York 2010-10-27 2010-10-27 07:00:00 148.0 361 +Chicago New York 2010-10-28 2010-10-28 07:00:00 2.0 361 +Chicago New York 2010-10-29 2010-10-29 07:00:00 -12.0 361 +Chicago New York 2010-10-31 2010-10-31 07:00:00 -25.0 361 +Chicago New York 2010-10-23 2010-10-23 07:00:00 3.0 384 +Chicago New York 2010-10-30 2010-10-30 07:00:00 -6.0 384 +Chicago New York 2010-10-23 2010-10-23 07:00:00 32.0 426 +Chicago New York 2010-10-30 2010-10-30 07:00:00 -10.0 426 +Chicago New York 2010-10-23 2010-10-23 07:00:00 1.0 650 +Chicago New York 2010-10-30 2010-10-30 07:00:00 -5.0 650 +Chicago New York 2010-10-20 2010-10-20 07:00:00 24.0 897 +Chicago New York 2010-10-21 2010-10-21 07:00:00 77.0 897 +Chicago New York 2010-10-22 2010-10-22 07:00:00 24.0 897 +Chicago New York 2010-10-24 2010-10-24 07:00:00 113.0 897 +Chicago New York 2010-10-25 2010-10-25 07:00:00 -1.0 897 +Chicago New York 2010-10-26 2010-10-26 07:00:00 0.0 897 +Chicago New York 2010-10-27 2010-10-27 07:00:00 -11.0 897 +PREHOOK: query: select fl_date, count(*) from flights_tiny_orc group by fl_date +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc +#### A masked pattern was here #### +POSTHOOK: query: select fl_date, count(*) from flights_tiny_orc group by fl_date +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc +#### A masked pattern was here #### +2010-10-20 11 +2010-10-21 12 +2010-10-22 11 +2010-10-23 12 +2010-10-24 12 +2010-10-25 12 +2010-10-26 13 +2010-10-27 11 +2010-10-28 12 +2010-10-29 12 +2010-10-30 11 +2010-10-31 8 +PREHOOK: query: explain +select * from flights_tiny_orc sort by fl_num, fl_date limit 25 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select * from flights_tiny_orc sort by fl_num, fl_date limit 25 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: flights_tiny_orc + Statistics: Num rows: 137 Data size: 39456 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), dest_city_name (type: string), fl_date (type: date), fl_time (type: timestamp), arr_delay (type: float), fl_num (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 137 Data size: 39456 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col5 (type: int), _col2 (type: date) + sort order: ++ + Statistics: Num rows: 137 Data size: 39456 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: timestamp), _col4 (type: float) + Execution mode: vectorized + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey1 (type: date), VALUE._col2 (type: timestamp), VALUE._col3 (type: float), KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 137 Data size: 39456 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 25 + Statistics: Num rows: 25 Data size: 7200 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col5 (type: int), _col2 (type: date) + sort order: ++ + Statistics: Num rows: 25 Data size: 7200 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: timestamp), _col4 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey1 (type: date), VALUE._col2 (type: timestamp), VALUE._col3 (type: float), KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 25 Data size: 7200 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 25 + Statistics: Num rows: 25 Data size: 7200 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 25 Data size: 7200 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: 25 + Processor Tree: + ListSink + +PREHOOK: query: select * from flights_tiny_orc sort by fl_num, fl_date limit 25 +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc +#### A masked pattern was here #### +POSTHOOK: query: select * from flights_tiny_orc sort by fl_num, fl_date limit 25 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc +#### A masked pattern was here #### +Baltimore New York 2010-10-23 2010-10-23 07:00:00 18.0 272 +Baltimore New York 2010-10-30 2010-10-30 07:00:00 14.0 272 +Chicago New York 2010-10-20 2010-10-20 07:00:00 42.0 361 +Chicago New York 2010-10-21 2010-10-21 07:00:00 142.0 361 +Chicago New York 2010-10-22 2010-10-22 07:00:00 2.0 361 +Chicago New York 2010-10-24 2010-10-24 07:00:00 10.0 361 +Chicago New York 2010-10-25 2010-10-25 07:00:00 31.0 361 +Chicago New York 2010-10-26 2010-10-26 07:00:00 12.0 361 +Chicago New York 2010-10-27 2010-10-27 07:00:00 148.0 361 +Chicago New York 2010-10-28 2010-10-28 07:00:00 2.0 361 +Chicago New York 2010-10-29 2010-10-29 07:00:00 -12.0 361 +Chicago New York 2010-10-31 2010-10-31 07:00:00 -25.0 361 +Chicago New York 2010-10-23 2010-10-23 07:00:00 3.0 384 +Chicago New York 2010-10-30 2010-10-30 07:00:00 -6.0 384 +Chicago New York 2010-10-23 2010-10-23 07:00:00 32.0 426 +Chicago New York 2010-10-30 2010-10-30 07:00:00 -10.0 426 +Chicago New York 2010-10-23 2010-10-23 07:00:00 1.0 650 +Chicago New York 2010-10-30 2010-10-30 07:00:00 -5.0 650 +Chicago New York 2010-10-20 2010-10-20 07:00:00 24.0 897 +Chicago New York 2010-10-21 2010-10-21 07:00:00 77.0 897 +Chicago New York 2010-10-22 2010-10-22 07:00:00 24.0 897 +Chicago New York 2010-10-24 2010-10-24 07:00:00 113.0 897 +Chicago New York 2010-10-25 2010-10-25 07:00:00 -1.0 897 +Chicago New York 2010-10-26 2010-10-26 07:00:00 0.0 897 +Chicago New York 2010-10-27 2010-10-27 07:00:00 -11.0 897 +PREHOOK: query: explain +select fl_date, count(*) from flights_tiny_orc group by fl_date +PREHOOK: type: QUERY +POSTHOOK: query: explain +select fl_date, count(*) from flights_tiny_orc group by fl_date +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: flights_tiny_orc + Statistics: Num rows: 137 Data size: 39456 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: fl_date (type: date) + outputColumnNames: fl_date + Statistics: Num rows: 137 Data size: 39456 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: fl_date (type: date) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 137 Data size: 39456 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: date) + sort order: + + Map-reduce partition columns: _col0 (type: date) + Statistics: Num rows: 137 Data size: 39456 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: date) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 68 Data size: 19584 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: date), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 68 Data size: 19584 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 68 Data size: 19584 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select fl_date, count(*) from flights_tiny_orc group by fl_date +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc +#### A masked pattern was here #### +POSTHOOK: query: select fl_date, count(*) from flights_tiny_orc group by fl_date +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc +#### A masked pattern was here #### +2010-10-20 11 +2010-10-21 12 +2010-10-22 11 +2010-10-23 12 +2010-10-24 12 +2010-10-25 12 +2010-10-26 13 +2010-10-27 11 +2010-10-28 12 +2010-10-29 12 +2010-10-30 11 +2010-10-31 8 +PREHOOK: query: CREATE TABLE flights_tiny_orc_partitioned_date ( + origin_city_name STRING, + dest_city_name STRING, + fl_time TIMESTAMP, + arr_delay FLOAT, + fl_num INT +) +PARTITIONED BY (fl_date DATE) +STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@flights_tiny_orc_partitioned_date +POSTHOOK: query: CREATE TABLE flights_tiny_orc_partitioned_date ( + origin_city_name STRING, + dest_city_name STRING, + fl_time TIMESTAMP, + arr_delay FLOAT, + fl_num INT +) +PARTITIONED BY (fl_date DATE) +STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@flights_tiny_orc_partitioned_date +PREHOOK: query: INSERT INTO TABLE flights_tiny_orc_partitioned_date +PARTITION (fl_date) +SELECT origin_city_name, dest_city_name, fl_time, arr_delay, fl_num, fl_date +FROM flights_tiny_orc +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc +PREHOOK: Output: default@flights_tiny_orc_partitioned_date +POSTHOOK: query: INSERT INTO TABLE flights_tiny_orc_partitioned_date +PARTITION (fl_date) +SELECT origin_city_name, dest_city_name, fl_time, arr_delay, fl_num, fl_date +FROM flights_tiny_orc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc +POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-20).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-20).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-20).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-20).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-20).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-21).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-21).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-21).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-21).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-21).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-22).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-22).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-22).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-22).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-22).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-23).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-23).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-23).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-23).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-23).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-24).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-24).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-24).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-24).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-24).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-25).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-25).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-25).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-25).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-25).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-26).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-26).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-26).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-26).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-26).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-27).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-27).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-27).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-27).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-27).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-28).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-28).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-28).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-28).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-28).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-29).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-29).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-29).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-29).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-29).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-30).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-30).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-30).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-30).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-30).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-31).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-31).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-31).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-31).fl_time SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_time, type:timestamp, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_date PARTITION(fl_date=2010-10-31).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +PREHOOK: query: select * from flights_tiny_orc_partitioned_date +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc_partitioned_date +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 +#### A masked pattern was here #### +POSTHOOK: query: select * from flights_tiny_orc_partitioned_date +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 +#### A masked pattern was here #### +Baltimore New York 2010-10-20 07:00:00 -30.0 1064 2010-10-20 +Baltimore New York 2010-10-20 07:00:00 23.0 1142 2010-10-20 +Baltimore New York 2010-10-20 07:00:00 6.0 1599 2010-10-20 +Chicago New York 2010-10-20 07:00:00 42.0 361 2010-10-20 +Chicago New York 2010-10-20 07:00:00 24.0 897 2010-10-20 +Chicago New York 2010-10-20 07:00:00 15.0 1531 2010-10-20 +Chicago New York 2010-10-20 07:00:00 -6.0 1610 2010-10-20 +Chicago New York 2010-10-20 07:00:00 -2.0 3198 2010-10-20 +Cleveland New York 2010-10-20 07:00:00 -8.0 2630 2010-10-20 +Cleveland New York 2010-10-20 07:00:00 -15.0 3014 2010-10-20 +Washington New York 2010-10-20 07:00:00 -2.0 7291 2010-10-20 +Baltimore New York 2010-10-21 07:00:00 17.0 1064 2010-10-21 +Baltimore New York 2010-10-21 07:00:00 105.0 1142 2010-10-21 +Baltimore New York 2010-10-21 07:00:00 28.0 1599 2010-10-21 +Chicago New York 2010-10-21 07:00:00 142.0 361 2010-10-21 +Chicago New York 2010-10-21 07:00:00 77.0 897 2010-10-21 +Chicago New York 2010-10-21 07:00:00 53.0 1531 2010-10-21 +Chicago New York 2010-10-21 07:00:00 -5.0 1610 2010-10-21 +Chicago New York 2010-10-21 07:00:00 51.0 3198 2010-10-21 +Cleveland New York 2010-10-21 07:00:00 3.0 2630 2010-10-21 +Cleveland New York 2010-10-21 07:00:00 29.0 2646 2010-10-21 +Cleveland New York 2010-10-21 07:00:00 72.0 3014 2010-10-21 +Washington New York 2010-10-21 07:00:00 22.0 7291 2010-10-21 +Baltimore New York 2010-10-22 07:00:00 -12.0 1064 2010-10-22 +Baltimore New York 2010-10-22 07:00:00 54.0 1142 2010-10-22 +Baltimore New York 2010-10-22 07:00:00 18.0 1599 2010-10-22 +Chicago New York 2010-10-22 07:00:00 2.0 361 2010-10-22 +Chicago New York 2010-10-22 07:00:00 24.0 897 2010-10-22 +Chicago New York 2010-10-22 07:00:00 16.0 1531 2010-10-22 +Chicago New York 2010-10-22 07:00:00 -6.0 1610 2010-10-22 +Chicago New York 2010-10-22 07:00:00 -11.0 3198 2010-10-22 +Cleveland New York 2010-10-22 07:00:00 1.0 2630 2010-10-22 +Cleveland New York 2010-10-22 07:00:00 -25.0 2646 2010-10-22 +Cleveland New York 2010-10-22 07:00:00 -3.0 3014 2010-10-22 +Baltimore New York 2010-10-23 07:00:00 18.0 272 2010-10-23 +Baltimore New York 2010-10-23 07:00:00 -10.0 1805 2010-10-23 +Baltimore New York 2010-10-23 07:00:00 6.0 3171 2010-10-23 +Chicago New York 2010-10-23 07:00:00 3.0 384 2010-10-23 +Chicago New York 2010-10-23 07:00:00 32.0 426 2010-10-23 +Chicago New York 2010-10-23 07:00:00 1.0 650 2010-10-23 +Chicago New York 2010-10-23 07:00:00 11.0 3085 2010-10-23 +Cleveland New York 2010-10-23 07:00:00 -21.0 2932 2010-10-23 +Washington New York 2010-10-23 07:00:00 -25.0 5832 2010-10-23 +Washington New York 2010-10-23 07:00:00 -21.0 5904 2010-10-23 +Washington New York 2010-10-23 07:00:00 -18.0 5917 2010-10-23 +Washington New York 2010-10-23 07:00:00 -16.0 7274 2010-10-23 +Baltimore New York 2010-10-24 07:00:00 12.0 1599 2010-10-24 +Baltimore New York 2010-10-24 07:00:00 20.0 2571 2010-10-24 +Chicago New York 2010-10-24 07:00:00 10.0 361 2010-10-24 +Chicago New York 2010-10-24 07:00:00 113.0 897 2010-10-24 +Chicago New York 2010-10-24 07:00:00 -5.0 1531 2010-10-24 +Chicago New York 2010-10-24 07:00:00 -17.0 1610 2010-10-24 +Chicago New York 2010-10-24 07:00:00 -3.0 3198 2010-10-24 +Cleveland New York 2010-10-24 07:00:00 5.0 2254 2010-10-24 +Cleveland New York 2010-10-24 07:00:00 -11.0 2630 2010-10-24 +Cleveland New York 2010-10-24 07:00:00 -20.0 2646 2010-10-24 +Cleveland New York 2010-10-24 07:00:00 -9.0 3014 2010-10-24 +Washington New York 2010-10-24 07:00:00 -26.0 7282 2010-10-24 +Baltimore New York 2010-10-25 07:00:00 -25.0 1064 2010-10-25 +Baltimore New York 2010-10-25 07:00:00 92.0 1142 2010-10-25 +Baltimore New York 2010-10-25 07:00:00 106.0 1599 2010-10-25 +Chicago New York 2010-10-25 07:00:00 31.0 361 2010-10-25 +Chicago New York 2010-10-25 07:00:00 -1.0 897 2010-10-25 +Chicago New York 2010-10-25 07:00:00 43.0 1531 2010-10-25 +Chicago New York 2010-10-25 07:00:00 6.0 1610 2010-10-25 +Chicago New York 2010-10-25 07:00:00 -16.0 3198 2010-10-25 +Cleveland New York 2010-10-25 07:00:00 -4.0 2630 2010-10-25 +Cleveland New York 2010-10-25 07:00:00 81.0 2646 2010-10-25 +Cleveland New York 2010-10-25 07:00:00 42.0 3014 2010-10-25 +Washington New York 2010-10-25 07:00:00 9.0 7291 2010-10-25 +Baltimore New York 2010-10-26 07:00:00 -22.0 1064 2010-10-26 +Baltimore New York 2010-10-26 07:00:00 123.0 1142 2010-10-26 +Baltimore New York 2010-10-26 07:00:00 90.0 1599 2010-10-26 +Chicago New York 2010-10-26 07:00:00 12.0 361 2010-10-26 +Chicago New York 2010-10-26 07:00:00 0.0 897 2010-10-26 +Chicago New York 2010-10-26 07:00:00 29.0 1531 2010-10-26 +Chicago New York 2010-10-26 07:00:00 -17.0 1610 2010-10-26 +Chicago New York 2010-10-26 07:00:00 6.0 3198 2010-10-26 +Cleveland New York 2010-10-26 07:00:00 4.0 2630 2010-10-26 +Cleveland New York 2010-10-26 07:00:00 -27.0 2646 2010-10-26 +Cleveland New York 2010-10-26 07:00:00 -11.0 2662 2010-10-26 +Cleveland New York 2010-10-26 07:00:00 13.0 3014 2010-10-26 +Washington New York 2010-10-26 07:00:00 4.0 7291 2010-10-26 +Baltimore New York 2010-10-27 07:00:00 -18.0 1064 2010-10-27 +Baltimore New York 2010-10-27 07:00:00 49.0 1142 2010-10-27 +Baltimore New York 2010-10-27 07:00:00 92.0 1599 2010-10-27 +Chicago New York 2010-10-27 07:00:00 148.0 361 2010-10-27 +Chicago New York 2010-10-27 07:00:00 -11.0 897 2010-10-27 +Chicago New York 2010-10-27 07:00:00 70.0 1531 2010-10-27 +Chicago New York 2010-10-27 07:00:00 8.0 1610 2010-10-27 +Chicago New York 2010-10-27 07:00:00 21.0 3198 2010-10-27 +Cleveland New York 2010-10-27 07:00:00 16.0 2630 2010-10-27 +Cleveland New York 2010-10-27 07:00:00 27.0 3014 2010-10-27 +Washington New York 2010-10-27 07:00:00 26.0 7291 2010-10-27 +Baltimore New York 2010-10-28 07:00:00 -4.0 1064 2010-10-28 +Baltimore New York 2010-10-28 07:00:00 -14.0 1142 2010-10-28 +Baltimore New York 2010-10-28 07:00:00 -14.0 1599 2010-10-28 +Chicago New York 2010-10-28 07:00:00 2.0 361 2010-10-28 +Chicago New York 2010-10-28 07:00:00 2.0 897 2010-10-28 +Chicago New York 2010-10-28 07:00:00 -11.0 1531 2010-10-28 +Chicago New York 2010-10-28 07:00:00 3.0 1610 2010-10-28 +Chicago New York 2010-10-28 07:00:00 -18.0 3198 2010-10-28 +Cleveland New York 2010-10-28 07:00:00 3.0 2630 2010-10-28 +Cleveland New York 2010-10-28 07:00:00 -6.0 2646 2010-10-28 +Cleveland New York 2010-10-28 07:00:00 1.0 3014 2010-10-28 +Washington New York 2010-10-28 07:00:00 45.0 7291 2010-10-28 +Baltimore New York 2010-10-29 07:00:00 -24.0 1064 2010-10-29 +Baltimore New York 2010-10-29 07:00:00 21.0 1142 2010-10-29 +Baltimore New York 2010-10-29 07:00:00 -2.0 1599 2010-10-29 +Chicago New York 2010-10-29 07:00:00 -12.0 361 2010-10-29 +Chicago New York 2010-10-29 07:00:00 -11.0 897 2010-10-29 +Chicago New York 2010-10-29 07:00:00 15.0 1531 2010-10-29 +Chicago New York 2010-10-29 07:00:00 -18.0 1610 2010-10-29 +Chicago New York 2010-10-29 07:00:00 -4.0 3198 2010-10-29 +Cleveland New York 2010-10-29 07:00:00 -4.0 2630 2010-10-29 +Cleveland New York 2010-10-29 07:00:00 -19.0 2646 2010-10-29 +Cleveland New York 2010-10-29 07:00:00 -12.0 3014 2010-10-29 +Washington New York 2010-10-29 07:00:00 1.0 7291 2010-10-29 +Baltimore New York 2010-10-30 07:00:00 14.0 272 2010-10-30 +Baltimore New York 2010-10-30 07:00:00 -1.0 1805 2010-10-30 +Baltimore New York 2010-10-30 07:00:00 5.0 3171 2010-10-30 +Chicago New York 2010-10-30 07:00:00 -6.0 384 2010-10-30 +Chicago New York 2010-10-30 07:00:00 -10.0 426 2010-10-30 +Chicago New York 2010-10-30 07:00:00 -5.0 650 2010-10-30 +Chicago New York 2010-10-30 07:00:00 -5.0 3085 2010-10-30 +Cleveland New York 2010-10-30 07:00:00 -23.0 2018 2010-10-30 +Cleveland New York 2010-10-30 07:00:00 -12.0 2932 2010-10-30 +Washington New York 2010-10-30 07:00:00 -27.0 5904 2010-10-30 +Washington New York 2010-10-30 07:00:00 -16.0 5917 2010-10-30 +Baltimore New York 2010-10-31 07:00:00 -1.0 1599 2010-10-31 +Baltimore New York 2010-10-31 07:00:00 -14.0 2571 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -25.0 361 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -18.0 897 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -4.0 1531 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -22.0 1610 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -15.0 3198 2010-10-31 +Washington New York 2010-10-31 07:00:00 -18.0 7282 2010-10-31 +PREHOOK: query: select * from flights_tiny_orc_partitioned_date sort by fl_num, fl_date limit 25 +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc_partitioned_date +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 +#### A masked pattern was here #### +POSTHOOK: query: select * from flights_tiny_orc_partitioned_date sort by fl_num, fl_date limit 25 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 +#### A masked pattern was here #### +Baltimore New York 2010-10-23 07:00:00 18.0 272 2010-10-23 +Baltimore New York 2010-10-30 07:00:00 14.0 272 2010-10-30 +Chicago New York 2010-10-20 07:00:00 42.0 361 2010-10-20 +Chicago New York 2010-10-21 07:00:00 142.0 361 2010-10-21 +Chicago New York 2010-10-22 07:00:00 2.0 361 2010-10-22 +Chicago New York 2010-10-24 07:00:00 10.0 361 2010-10-24 +Chicago New York 2010-10-25 07:00:00 31.0 361 2010-10-25 +Chicago New York 2010-10-26 07:00:00 12.0 361 2010-10-26 +Chicago New York 2010-10-27 07:00:00 148.0 361 2010-10-27 +Chicago New York 2010-10-28 07:00:00 2.0 361 2010-10-28 +Chicago New York 2010-10-29 07:00:00 -12.0 361 2010-10-29 +Chicago New York 2010-10-31 07:00:00 -25.0 361 2010-10-31 +Chicago New York 2010-10-23 07:00:00 3.0 384 2010-10-23 +Chicago New York 2010-10-30 07:00:00 -6.0 384 2010-10-30 +Chicago New York 2010-10-23 07:00:00 32.0 426 2010-10-23 +Chicago New York 2010-10-30 07:00:00 -10.0 426 2010-10-30 +Chicago New York 2010-10-23 07:00:00 1.0 650 2010-10-23 +Chicago New York 2010-10-30 07:00:00 -5.0 650 2010-10-30 +Chicago New York 2010-10-20 07:00:00 24.0 897 2010-10-20 +Chicago New York 2010-10-21 07:00:00 77.0 897 2010-10-21 +Chicago New York 2010-10-22 07:00:00 24.0 897 2010-10-22 +Chicago New York 2010-10-24 07:00:00 113.0 897 2010-10-24 +Chicago New York 2010-10-25 07:00:00 -1.0 897 2010-10-25 +Chicago New York 2010-10-26 07:00:00 0.0 897 2010-10-26 +Chicago New York 2010-10-27 07:00:00 -11.0 897 2010-10-27 +PREHOOK: query: select fl_date, count(*) from flights_tiny_orc_partitioned_date group by fl_date +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc_partitioned_date +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 +#### A masked pattern was here #### +POSTHOOK: query: select fl_date, count(*) from flights_tiny_orc_partitioned_date group by fl_date +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 +#### A masked pattern was here #### +2010-10-20 11 +2010-10-21 12 +2010-10-22 11 +2010-10-23 12 +2010-10-24 12 +2010-10-25 12 +2010-10-26 13 +2010-10-27 11 +2010-10-28 12 +2010-10-29 12 +2010-10-30 11 +2010-10-31 8 +PREHOOK: query: explain +select * from flights_tiny_orc_partitioned_date +PREHOOK: type: QUERY +POSTHOOK: query: explain +select * from flights_tiny_orc_partitioned_date +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: flights_tiny_orc_partitioned_date + Statistics: Num rows: 137 Data size: 31776 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), dest_city_name (type: string), fl_time (type: timestamp), arr_delay (type: float), fl_num (type: int), fl_date (type: date) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 137 Data size: 31776 Basic stats: COMPLETE Column stats: NONE + ListSink + +PREHOOK: query: select * from flights_tiny_orc_partitioned_date +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc_partitioned_date +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 +#### A masked pattern was here #### +POSTHOOK: query: select * from flights_tiny_orc_partitioned_date +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 +#### A masked pattern was here #### +Baltimore New York 2010-10-20 07:00:00 -30.0 1064 2010-10-20 +Baltimore New York 2010-10-20 07:00:00 23.0 1142 2010-10-20 +Baltimore New York 2010-10-20 07:00:00 6.0 1599 2010-10-20 +Chicago New York 2010-10-20 07:00:00 42.0 361 2010-10-20 +Chicago New York 2010-10-20 07:00:00 24.0 897 2010-10-20 +Chicago New York 2010-10-20 07:00:00 15.0 1531 2010-10-20 +Chicago New York 2010-10-20 07:00:00 -6.0 1610 2010-10-20 +Chicago New York 2010-10-20 07:00:00 -2.0 3198 2010-10-20 +Cleveland New York 2010-10-20 07:00:00 -8.0 2630 2010-10-20 +Cleveland New York 2010-10-20 07:00:00 -15.0 3014 2010-10-20 +Washington New York 2010-10-20 07:00:00 -2.0 7291 2010-10-20 +Baltimore New York 2010-10-21 07:00:00 17.0 1064 2010-10-21 +Baltimore New York 2010-10-21 07:00:00 105.0 1142 2010-10-21 +Baltimore New York 2010-10-21 07:00:00 28.0 1599 2010-10-21 +Chicago New York 2010-10-21 07:00:00 142.0 361 2010-10-21 +Chicago New York 2010-10-21 07:00:00 77.0 897 2010-10-21 +Chicago New York 2010-10-21 07:00:00 53.0 1531 2010-10-21 +Chicago New York 2010-10-21 07:00:00 -5.0 1610 2010-10-21 +Chicago New York 2010-10-21 07:00:00 51.0 3198 2010-10-21 +Cleveland New York 2010-10-21 07:00:00 3.0 2630 2010-10-21 +Cleveland New York 2010-10-21 07:00:00 29.0 2646 2010-10-21 +Cleveland New York 2010-10-21 07:00:00 72.0 3014 2010-10-21 +Washington New York 2010-10-21 07:00:00 22.0 7291 2010-10-21 +Baltimore New York 2010-10-22 07:00:00 -12.0 1064 2010-10-22 +Baltimore New York 2010-10-22 07:00:00 54.0 1142 2010-10-22 +Baltimore New York 2010-10-22 07:00:00 18.0 1599 2010-10-22 +Chicago New York 2010-10-22 07:00:00 2.0 361 2010-10-22 +Chicago New York 2010-10-22 07:00:00 24.0 897 2010-10-22 +Chicago New York 2010-10-22 07:00:00 16.0 1531 2010-10-22 +Chicago New York 2010-10-22 07:00:00 -6.0 1610 2010-10-22 +Chicago New York 2010-10-22 07:00:00 -11.0 3198 2010-10-22 +Cleveland New York 2010-10-22 07:00:00 1.0 2630 2010-10-22 +Cleveland New York 2010-10-22 07:00:00 -25.0 2646 2010-10-22 +Cleveland New York 2010-10-22 07:00:00 -3.0 3014 2010-10-22 +Baltimore New York 2010-10-23 07:00:00 18.0 272 2010-10-23 +Baltimore New York 2010-10-23 07:00:00 -10.0 1805 2010-10-23 +Baltimore New York 2010-10-23 07:00:00 6.0 3171 2010-10-23 +Chicago New York 2010-10-23 07:00:00 3.0 384 2010-10-23 +Chicago New York 2010-10-23 07:00:00 32.0 426 2010-10-23 +Chicago New York 2010-10-23 07:00:00 1.0 650 2010-10-23 +Chicago New York 2010-10-23 07:00:00 11.0 3085 2010-10-23 +Cleveland New York 2010-10-23 07:00:00 -21.0 2932 2010-10-23 +Washington New York 2010-10-23 07:00:00 -25.0 5832 2010-10-23 +Washington New York 2010-10-23 07:00:00 -21.0 5904 2010-10-23 +Washington New York 2010-10-23 07:00:00 -18.0 5917 2010-10-23 +Washington New York 2010-10-23 07:00:00 -16.0 7274 2010-10-23 +Baltimore New York 2010-10-24 07:00:00 12.0 1599 2010-10-24 +Baltimore New York 2010-10-24 07:00:00 20.0 2571 2010-10-24 +Chicago New York 2010-10-24 07:00:00 10.0 361 2010-10-24 +Chicago New York 2010-10-24 07:00:00 113.0 897 2010-10-24 +Chicago New York 2010-10-24 07:00:00 -5.0 1531 2010-10-24 +Chicago New York 2010-10-24 07:00:00 -17.0 1610 2010-10-24 +Chicago New York 2010-10-24 07:00:00 -3.0 3198 2010-10-24 +Cleveland New York 2010-10-24 07:00:00 5.0 2254 2010-10-24 +Cleveland New York 2010-10-24 07:00:00 -11.0 2630 2010-10-24 +Cleveland New York 2010-10-24 07:00:00 -20.0 2646 2010-10-24 +Cleveland New York 2010-10-24 07:00:00 -9.0 3014 2010-10-24 +Washington New York 2010-10-24 07:00:00 -26.0 7282 2010-10-24 +Baltimore New York 2010-10-25 07:00:00 -25.0 1064 2010-10-25 +Baltimore New York 2010-10-25 07:00:00 92.0 1142 2010-10-25 +Baltimore New York 2010-10-25 07:00:00 106.0 1599 2010-10-25 +Chicago New York 2010-10-25 07:00:00 31.0 361 2010-10-25 +Chicago New York 2010-10-25 07:00:00 -1.0 897 2010-10-25 +Chicago New York 2010-10-25 07:00:00 43.0 1531 2010-10-25 +Chicago New York 2010-10-25 07:00:00 6.0 1610 2010-10-25 +Chicago New York 2010-10-25 07:00:00 -16.0 3198 2010-10-25 +Cleveland New York 2010-10-25 07:00:00 -4.0 2630 2010-10-25 +Cleveland New York 2010-10-25 07:00:00 81.0 2646 2010-10-25 +Cleveland New York 2010-10-25 07:00:00 42.0 3014 2010-10-25 +Washington New York 2010-10-25 07:00:00 9.0 7291 2010-10-25 +Baltimore New York 2010-10-26 07:00:00 -22.0 1064 2010-10-26 +Baltimore New York 2010-10-26 07:00:00 123.0 1142 2010-10-26 +Baltimore New York 2010-10-26 07:00:00 90.0 1599 2010-10-26 +Chicago New York 2010-10-26 07:00:00 12.0 361 2010-10-26 +Chicago New York 2010-10-26 07:00:00 0.0 897 2010-10-26 +Chicago New York 2010-10-26 07:00:00 29.0 1531 2010-10-26 +Chicago New York 2010-10-26 07:00:00 -17.0 1610 2010-10-26 +Chicago New York 2010-10-26 07:00:00 6.0 3198 2010-10-26 +Cleveland New York 2010-10-26 07:00:00 4.0 2630 2010-10-26 +Cleveland New York 2010-10-26 07:00:00 -27.0 2646 2010-10-26 +Cleveland New York 2010-10-26 07:00:00 -11.0 2662 2010-10-26 +Cleveland New York 2010-10-26 07:00:00 13.0 3014 2010-10-26 +Washington New York 2010-10-26 07:00:00 4.0 7291 2010-10-26 +Baltimore New York 2010-10-27 07:00:00 -18.0 1064 2010-10-27 +Baltimore New York 2010-10-27 07:00:00 49.0 1142 2010-10-27 +Baltimore New York 2010-10-27 07:00:00 92.0 1599 2010-10-27 +Chicago New York 2010-10-27 07:00:00 148.0 361 2010-10-27 +Chicago New York 2010-10-27 07:00:00 -11.0 897 2010-10-27 +Chicago New York 2010-10-27 07:00:00 70.0 1531 2010-10-27 +Chicago New York 2010-10-27 07:00:00 8.0 1610 2010-10-27 +Chicago New York 2010-10-27 07:00:00 21.0 3198 2010-10-27 +Cleveland New York 2010-10-27 07:00:00 16.0 2630 2010-10-27 +Cleveland New York 2010-10-27 07:00:00 27.0 3014 2010-10-27 +Washington New York 2010-10-27 07:00:00 26.0 7291 2010-10-27 +Baltimore New York 2010-10-28 07:00:00 -4.0 1064 2010-10-28 +Baltimore New York 2010-10-28 07:00:00 -14.0 1142 2010-10-28 +Baltimore New York 2010-10-28 07:00:00 -14.0 1599 2010-10-28 +Chicago New York 2010-10-28 07:00:00 2.0 361 2010-10-28 +Chicago New York 2010-10-28 07:00:00 2.0 897 2010-10-28 +Chicago New York 2010-10-28 07:00:00 -11.0 1531 2010-10-28 +Chicago New York 2010-10-28 07:00:00 3.0 1610 2010-10-28 +Chicago New York 2010-10-28 07:00:00 -18.0 3198 2010-10-28 +Cleveland New York 2010-10-28 07:00:00 3.0 2630 2010-10-28 +Cleveland New York 2010-10-28 07:00:00 -6.0 2646 2010-10-28 +Cleveland New York 2010-10-28 07:00:00 1.0 3014 2010-10-28 +Washington New York 2010-10-28 07:00:00 45.0 7291 2010-10-28 +Baltimore New York 2010-10-29 07:00:00 -24.0 1064 2010-10-29 +Baltimore New York 2010-10-29 07:00:00 21.0 1142 2010-10-29 +Baltimore New York 2010-10-29 07:00:00 -2.0 1599 2010-10-29 +Chicago New York 2010-10-29 07:00:00 -12.0 361 2010-10-29 +Chicago New York 2010-10-29 07:00:00 -11.0 897 2010-10-29 +Chicago New York 2010-10-29 07:00:00 15.0 1531 2010-10-29 +Chicago New York 2010-10-29 07:00:00 -18.0 1610 2010-10-29 +Chicago New York 2010-10-29 07:00:00 -4.0 3198 2010-10-29 +Cleveland New York 2010-10-29 07:00:00 -4.0 2630 2010-10-29 +Cleveland New York 2010-10-29 07:00:00 -19.0 2646 2010-10-29 +Cleveland New York 2010-10-29 07:00:00 -12.0 3014 2010-10-29 +Washington New York 2010-10-29 07:00:00 1.0 7291 2010-10-29 +Baltimore New York 2010-10-30 07:00:00 14.0 272 2010-10-30 +Baltimore New York 2010-10-30 07:00:00 -1.0 1805 2010-10-30 +Baltimore New York 2010-10-30 07:00:00 5.0 3171 2010-10-30 +Chicago New York 2010-10-30 07:00:00 -6.0 384 2010-10-30 +Chicago New York 2010-10-30 07:00:00 -10.0 426 2010-10-30 +Chicago New York 2010-10-30 07:00:00 -5.0 650 2010-10-30 +Chicago New York 2010-10-30 07:00:00 -5.0 3085 2010-10-30 +Cleveland New York 2010-10-30 07:00:00 -23.0 2018 2010-10-30 +Cleveland New York 2010-10-30 07:00:00 -12.0 2932 2010-10-30 +Washington New York 2010-10-30 07:00:00 -27.0 5904 2010-10-30 +Washington New York 2010-10-30 07:00:00 -16.0 5917 2010-10-30 +Baltimore New York 2010-10-31 07:00:00 -1.0 1599 2010-10-31 +Baltimore New York 2010-10-31 07:00:00 -14.0 2571 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -25.0 361 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -18.0 897 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -4.0 1531 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -22.0 1610 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -15.0 3198 2010-10-31 +Washington New York 2010-10-31 07:00:00 -18.0 7282 2010-10-31 +PREHOOK: query: explain +select * from flights_tiny_orc_partitioned_date sort by fl_num, fl_date limit 25 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select * from flights_tiny_orc_partitioned_date sort by fl_num, fl_date limit 25 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: flights_tiny_orc_partitioned_date + Statistics: Num rows: 137 Data size: 31776 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), dest_city_name (type: string), fl_time (type: timestamp), arr_delay (type: float), fl_num (type: int), fl_date (type: date) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 137 Data size: 31776 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: int), _col5 (type: date) + sort order: ++ + Statistics: Num rows: 137 Data size: 31776 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: timestamp), _col3 (type: float) + Execution mode: vectorized + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: timestamp), VALUE._col3 (type: float), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: date) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 137 Data size: 31776 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 25 + Statistics: Num rows: 25 Data size: 5775 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col4 (type: int), _col5 (type: date) + sort order: ++ + Statistics: Num rows: 25 Data size: 5775 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: timestamp), _col3 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: timestamp), VALUE._col3 (type: float), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: date) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 25 Data size: 5775 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 25 + Statistics: Num rows: 25 Data size: 5775 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 25 Data size: 5775 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: 25 + Processor Tree: + ListSink + +PREHOOK: query: select * from flights_tiny_orc_partitioned_date sort by fl_num, fl_date limit 25 +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc_partitioned_date +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 +#### A masked pattern was here #### +POSTHOOK: query: select * from flights_tiny_orc_partitioned_date sort by fl_num, fl_date limit 25 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 +#### A masked pattern was here #### +Baltimore New York 2010-10-23 07:00:00 18.0 272 2010-10-23 +Baltimore New York 2010-10-30 07:00:00 14.0 272 2010-10-30 +Chicago New York 2010-10-20 07:00:00 42.0 361 2010-10-20 +Chicago New York 2010-10-21 07:00:00 142.0 361 2010-10-21 +Chicago New York 2010-10-22 07:00:00 2.0 361 2010-10-22 +Chicago New York 2010-10-24 07:00:00 10.0 361 2010-10-24 +Chicago New York 2010-10-25 07:00:00 31.0 361 2010-10-25 +Chicago New York 2010-10-26 07:00:00 12.0 361 2010-10-26 +Chicago New York 2010-10-27 07:00:00 148.0 361 2010-10-27 +Chicago New York 2010-10-28 07:00:00 2.0 361 2010-10-28 +Chicago New York 2010-10-29 07:00:00 -12.0 361 2010-10-29 +Chicago New York 2010-10-31 07:00:00 -25.0 361 2010-10-31 +Chicago New York 2010-10-23 07:00:00 3.0 384 2010-10-23 +Chicago New York 2010-10-30 07:00:00 -6.0 384 2010-10-30 +Chicago New York 2010-10-23 07:00:00 32.0 426 2010-10-23 +Chicago New York 2010-10-30 07:00:00 -10.0 426 2010-10-30 +Chicago New York 2010-10-23 07:00:00 1.0 650 2010-10-23 +Chicago New York 2010-10-30 07:00:00 -5.0 650 2010-10-30 +Chicago New York 2010-10-20 07:00:00 24.0 897 2010-10-20 +Chicago New York 2010-10-21 07:00:00 77.0 897 2010-10-21 +Chicago New York 2010-10-22 07:00:00 24.0 897 2010-10-22 +Chicago New York 2010-10-24 07:00:00 113.0 897 2010-10-24 +Chicago New York 2010-10-25 07:00:00 -1.0 897 2010-10-25 +Chicago New York 2010-10-26 07:00:00 0.0 897 2010-10-26 +Chicago New York 2010-10-27 07:00:00 -11.0 897 2010-10-27 +PREHOOK: query: explain +select fl_date, count(*) from flights_tiny_orc_partitioned_date group by fl_date +PREHOOK: type: QUERY +POSTHOOK: query: explain +select fl_date, count(*) from flights_tiny_orc_partitioned_date group by fl_date +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: flights_tiny_orc_partitioned_date + Statistics: Num rows: 137 Data size: 31776 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: fl_date (type: date) + outputColumnNames: fl_date + Statistics: Num rows: 137 Data size: 31776 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: fl_date (type: date) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 137 Data size: 31776 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: date) + sort order: + + Map-reduce partition columns: _col0 (type: date) + Statistics: Num rows: 137 Data size: 31776 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: date) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 68 Data size: 15772 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: date), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 68 Data size: 15772 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 68 Data size: 15772 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select fl_date, count(*) from flights_tiny_orc_partitioned_date group by fl_date +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc_partitioned_date +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 +PREHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 +#### A masked pattern was here #### +POSTHOOK: query: select fl_date, count(*) from flights_tiny_orc_partitioned_date group by fl_date +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-20 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-21 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-22 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-23 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-24 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-25 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-26 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-27 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-28 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 +#### A masked pattern was here #### +2010-10-20 11 +2010-10-21 12 +2010-10-22 11 +2010-10-23 12 +2010-10-24 12 +2010-10-25 12 +2010-10-26 13 +2010-10-27 11 +2010-10-28 12 +2010-10-29 12 +2010-10-30 11 +2010-10-31 8 +PREHOOK: query: CREATE TABLE flights_tiny_orc_partitioned_timestamp ( + origin_city_name STRING, + dest_city_name STRING, + fl_date DATE, + arr_delay FLOAT, + fl_num INT +) +PARTITIONED BY (fl_time TIMESTAMP) +STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@flights_tiny_orc_partitioned_timestamp +POSTHOOK: query: CREATE TABLE flights_tiny_orc_partitioned_timestamp ( + origin_city_name STRING, + dest_city_name STRING, + fl_date DATE, + arr_delay FLOAT, + fl_num INT +) +PARTITIONED BY (fl_time TIMESTAMP) +STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp +PREHOOK: query: INSERT INTO TABLE flights_tiny_orc_partitioned_timestamp +PARTITION (fl_time) +SELECT origin_city_name, dest_city_name, fl_date, arr_delay, fl_num, fl_time +FROM flights_tiny_orc +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc +PREHOOK: Output: default@flights_tiny_orc_partitioned_timestamp +POSTHOOK: query: INSERT INTO TABLE flights_tiny_orc_partitioned_timestamp +PARTITION (fl_time) +SELECT origin_city_name, dest_city_name, fl_date, arr_delay, fl_num, fl_time +FROM flights_tiny_orc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc +POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20 07%3A00%3A00 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21 07%3A00%3A00 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22 07%3A00%3A00 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23 07%3A00%3A00 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24 07%3A00%3A00 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25 07%3A00%3A00 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26 07%3A00%3A00 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27 07%3A00%3A00 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28 07%3A00%3A00 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29 07%3A00%3A00 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 +POSTHOOK: Output: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-20 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-20 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-20 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-20 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-20 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-21 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-21 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-21 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-21 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-21 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-22 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-22 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-22 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-22 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-22 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-23 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-23 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-23 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-23 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-23 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-24 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-24 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-24 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-24 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-24 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-25 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-25 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-25 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-25 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-25 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-26 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-26 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-26 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-26 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-26 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-27 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-27 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-27 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-27 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-27 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-28 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-28 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-28 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-28 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-28 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-29 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-29 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-29 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-29 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-29 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-30 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-30 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-30 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-30 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-30 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-31 07:00:00).arr_delay SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:arr_delay, type:float, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-31 07:00:00).dest_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:dest_city_name, type:string, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-31 07:00:00).fl_date SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_date, type:date, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-31 07:00:00).fl_num SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:fl_num, type:int, comment:null), ] +POSTHOOK: Lineage: flights_tiny_orc_partitioned_timestamp PARTITION(fl_time=2010-10-31 07:00:00).origin_city_name SIMPLE [(flights_tiny_orc)flights_tiny_orc.FieldSchema(name:origin_city_name, type:string, comment:null), ] +PREHOOK: query: select * from flights_tiny_orc_partitioned_timestamp +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 +#### A masked pattern was here #### +POSTHOOK: query: select * from flights_tiny_orc_partitioned_timestamp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 +#### A masked pattern was here #### +Baltimore New York 2010-10-20 -30.0 1064 2010-10-20 07:00:00 +Baltimore New York 2010-10-20 23.0 1142 2010-10-20 07:00:00 +Baltimore New York 2010-10-20 6.0 1599 2010-10-20 07:00:00 +Chicago New York 2010-10-20 42.0 361 2010-10-20 07:00:00 +Chicago New York 2010-10-20 24.0 897 2010-10-20 07:00:00 +Chicago New York 2010-10-20 15.0 1531 2010-10-20 07:00:00 +Chicago New York 2010-10-20 -6.0 1610 2010-10-20 07:00:00 +Chicago New York 2010-10-20 -2.0 3198 2010-10-20 07:00:00 +Cleveland New York 2010-10-20 -8.0 2630 2010-10-20 07:00:00 +Cleveland New York 2010-10-20 -15.0 3014 2010-10-20 07:00:00 +Washington New York 2010-10-20 -2.0 7291 2010-10-20 07:00:00 +Baltimore New York 2010-10-21 17.0 1064 2010-10-21 07:00:00 +Baltimore New York 2010-10-21 105.0 1142 2010-10-21 07:00:00 +Baltimore New York 2010-10-21 28.0 1599 2010-10-21 07:00:00 +Chicago New York 2010-10-21 142.0 361 2010-10-21 07:00:00 +Chicago New York 2010-10-21 77.0 897 2010-10-21 07:00:00 +Chicago New York 2010-10-21 53.0 1531 2010-10-21 07:00:00 +Chicago New York 2010-10-21 -5.0 1610 2010-10-21 07:00:00 +Chicago New York 2010-10-21 51.0 3198 2010-10-21 07:00:00 +Cleveland New York 2010-10-21 3.0 2630 2010-10-21 07:00:00 +Cleveland New York 2010-10-21 29.0 2646 2010-10-21 07:00:00 +Cleveland New York 2010-10-21 72.0 3014 2010-10-21 07:00:00 +Washington New York 2010-10-21 22.0 7291 2010-10-21 07:00:00 +Baltimore New York 2010-10-22 -12.0 1064 2010-10-22 07:00:00 +Baltimore New York 2010-10-22 54.0 1142 2010-10-22 07:00:00 +Baltimore New York 2010-10-22 18.0 1599 2010-10-22 07:00:00 +Chicago New York 2010-10-22 2.0 361 2010-10-22 07:00:00 +Chicago New York 2010-10-22 24.0 897 2010-10-22 07:00:00 +Chicago New York 2010-10-22 16.0 1531 2010-10-22 07:00:00 +Chicago New York 2010-10-22 -6.0 1610 2010-10-22 07:00:00 +Chicago New York 2010-10-22 -11.0 3198 2010-10-22 07:00:00 +Cleveland New York 2010-10-22 1.0 2630 2010-10-22 07:00:00 +Cleveland New York 2010-10-22 -25.0 2646 2010-10-22 07:00:00 +Cleveland New York 2010-10-22 -3.0 3014 2010-10-22 07:00:00 +Baltimore New York 2010-10-23 18.0 272 2010-10-23 07:00:00 +Baltimore New York 2010-10-23 -10.0 1805 2010-10-23 07:00:00 +Baltimore New York 2010-10-23 6.0 3171 2010-10-23 07:00:00 +Chicago New York 2010-10-23 3.0 384 2010-10-23 07:00:00 +Chicago New York 2010-10-23 32.0 426 2010-10-23 07:00:00 +Chicago New York 2010-10-23 1.0 650 2010-10-23 07:00:00 +Chicago New York 2010-10-23 11.0 3085 2010-10-23 07:00:00 +Cleveland New York 2010-10-23 -21.0 2932 2010-10-23 07:00:00 +Washington New York 2010-10-23 -25.0 5832 2010-10-23 07:00:00 +Washington New York 2010-10-23 -21.0 5904 2010-10-23 07:00:00 +Washington New York 2010-10-23 -18.0 5917 2010-10-23 07:00:00 +Washington New York 2010-10-23 -16.0 7274 2010-10-23 07:00:00 +Baltimore New York 2010-10-24 12.0 1599 2010-10-24 07:00:00 +Baltimore New York 2010-10-24 20.0 2571 2010-10-24 07:00:00 +Chicago New York 2010-10-24 10.0 361 2010-10-24 07:00:00 +Chicago New York 2010-10-24 113.0 897 2010-10-24 07:00:00 +Chicago New York 2010-10-24 -5.0 1531 2010-10-24 07:00:00 +Chicago New York 2010-10-24 -17.0 1610 2010-10-24 07:00:00 +Chicago New York 2010-10-24 -3.0 3198 2010-10-24 07:00:00 +Cleveland New York 2010-10-24 5.0 2254 2010-10-24 07:00:00 +Cleveland New York 2010-10-24 -11.0 2630 2010-10-24 07:00:00 +Cleveland New York 2010-10-24 -20.0 2646 2010-10-24 07:00:00 +Cleveland New York 2010-10-24 -9.0 3014 2010-10-24 07:00:00 +Washington New York 2010-10-24 -26.0 7282 2010-10-24 07:00:00 +Baltimore New York 2010-10-25 -25.0 1064 2010-10-25 07:00:00 +Baltimore New York 2010-10-25 92.0 1142 2010-10-25 07:00:00 +Baltimore New York 2010-10-25 106.0 1599 2010-10-25 07:00:00 +Chicago New York 2010-10-25 31.0 361 2010-10-25 07:00:00 +Chicago New York 2010-10-25 -1.0 897 2010-10-25 07:00:00 +Chicago New York 2010-10-25 43.0 1531 2010-10-25 07:00:00 +Chicago New York 2010-10-25 6.0 1610 2010-10-25 07:00:00 +Chicago New York 2010-10-25 -16.0 3198 2010-10-25 07:00:00 +Cleveland New York 2010-10-25 -4.0 2630 2010-10-25 07:00:00 +Cleveland New York 2010-10-25 81.0 2646 2010-10-25 07:00:00 +Cleveland New York 2010-10-25 42.0 3014 2010-10-25 07:00:00 +Washington New York 2010-10-25 9.0 7291 2010-10-25 07:00:00 +Baltimore New York 2010-10-26 -22.0 1064 2010-10-26 07:00:00 +Baltimore New York 2010-10-26 123.0 1142 2010-10-26 07:00:00 +Baltimore New York 2010-10-26 90.0 1599 2010-10-26 07:00:00 +Chicago New York 2010-10-26 12.0 361 2010-10-26 07:00:00 +Chicago New York 2010-10-26 0.0 897 2010-10-26 07:00:00 +Chicago New York 2010-10-26 29.0 1531 2010-10-26 07:00:00 +Chicago New York 2010-10-26 -17.0 1610 2010-10-26 07:00:00 +Chicago New York 2010-10-26 6.0 3198 2010-10-26 07:00:00 +Cleveland New York 2010-10-26 4.0 2630 2010-10-26 07:00:00 +Cleveland New York 2010-10-26 -27.0 2646 2010-10-26 07:00:00 +Cleveland New York 2010-10-26 -11.0 2662 2010-10-26 07:00:00 +Cleveland New York 2010-10-26 13.0 3014 2010-10-26 07:00:00 +Washington New York 2010-10-26 4.0 7291 2010-10-26 07:00:00 +Baltimore New York 2010-10-27 -18.0 1064 2010-10-27 07:00:00 +Baltimore New York 2010-10-27 49.0 1142 2010-10-27 07:00:00 +Baltimore New York 2010-10-27 92.0 1599 2010-10-27 07:00:00 +Chicago New York 2010-10-27 148.0 361 2010-10-27 07:00:00 +Chicago New York 2010-10-27 -11.0 897 2010-10-27 07:00:00 +Chicago New York 2010-10-27 70.0 1531 2010-10-27 07:00:00 +Chicago New York 2010-10-27 8.0 1610 2010-10-27 07:00:00 +Chicago New York 2010-10-27 21.0 3198 2010-10-27 07:00:00 +Cleveland New York 2010-10-27 16.0 2630 2010-10-27 07:00:00 +Cleveland New York 2010-10-27 27.0 3014 2010-10-27 07:00:00 +Washington New York 2010-10-27 26.0 7291 2010-10-27 07:00:00 +Baltimore New York 2010-10-28 -4.0 1064 2010-10-28 07:00:00 +Baltimore New York 2010-10-28 -14.0 1142 2010-10-28 07:00:00 +Baltimore New York 2010-10-28 -14.0 1599 2010-10-28 07:00:00 +Chicago New York 2010-10-28 2.0 361 2010-10-28 07:00:00 +Chicago New York 2010-10-28 2.0 897 2010-10-28 07:00:00 +Chicago New York 2010-10-28 -11.0 1531 2010-10-28 07:00:00 +Chicago New York 2010-10-28 3.0 1610 2010-10-28 07:00:00 +Chicago New York 2010-10-28 -18.0 3198 2010-10-28 07:00:00 +Cleveland New York 2010-10-28 3.0 2630 2010-10-28 07:00:00 +Cleveland New York 2010-10-28 -6.0 2646 2010-10-28 07:00:00 +Cleveland New York 2010-10-28 1.0 3014 2010-10-28 07:00:00 +Washington New York 2010-10-28 45.0 7291 2010-10-28 07:00:00 +Baltimore New York 2010-10-29 -24.0 1064 2010-10-29 07:00:00 +Baltimore New York 2010-10-29 21.0 1142 2010-10-29 07:00:00 +Baltimore New York 2010-10-29 -2.0 1599 2010-10-29 07:00:00 +Chicago New York 2010-10-29 -12.0 361 2010-10-29 07:00:00 +Chicago New York 2010-10-29 -11.0 897 2010-10-29 07:00:00 +Chicago New York 2010-10-29 15.0 1531 2010-10-29 07:00:00 +Chicago New York 2010-10-29 -18.0 1610 2010-10-29 07:00:00 +Chicago New York 2010-10-29 -4.0 3198 2010-10-29 07:00:00 +Cleveland New York 2010-10-29 -4.0 2630 2010-10-29 07:00:00 +Cleveland New York 2010-10-29 -19.0 2646 2010-10-29 07:00:00 +Cleveland New York 2010-10-29 -12.0 3014 2010-10-29 07:00:00 +Washington New York 2010-10-29 1.0 7291 2010-10-29 07:00:00 +Baltimore New York 2010-10-30 14.0 272 2010-10-30 07:00:00 +Baltimore New York 2010-10-30 -1.0 1805 2010-10-30 07:00:00 +Baltimore New York 2010-10-30 5.0 3171 2010-10-30 07:00:00 +Chicago New York 2010-10-30 -6.0 384 2010-10-30 07:00:00 +Chicago New York 2010-10-30 -10.0 426 2010-10-30 07:00:00 +Chicago New York 2010-10-30 -5.0 650 2010-10-30 07:00:00 +Chicago New York 2010-10-30 -5.0 3085 2010-10-30 07:00:00 +Cleveland New York 2010-10-30 -23.0 2018 2010-10-30 07:00:00 +Cleveland New York 2010-10-30 -12.0 2932 2010-10-30 07:00:00 +Washington New York 2010-10-30 -27.0 5904 2010-10-30 07:00:00 +Washington New York 2010-10-30 -16.0 5917 2010-10-30 07:00:00 +Baltimore New York 2010-10-31 -1.0 1599 2010-10-31 07:00:00 +Baltimore New York 2010-10-31 -14.0 2571 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -25.0 361 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -18.0 897 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -4.0 1531 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -22.0 1610 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -15.0 3198 2010-10-31 07:00:00 +Washington New York 2010-10-31 -18.0 7282 2010-10-31 07:00:00 +PREHOOK: query: select * from flights_tiny_orc_partitioned_timestamp sort by fl_num, fl_time limit 25 +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 +#### A masked pattern was here #### +POSTHOOK: query: select * from flights_tiny_orc_partitioned_timestamp sort by fl_num, fl_time limit 25 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 +#### A masked pattern was here #### +Baltimore New York 2010-10-23 18.0 272 2010-10-23 07:00:00 +Baltimore New York 2010-10-30 14.0 272 2010-10-30 07:00:00 +Chicago New York 2010-10-20 42.0 361 2010-10-20 07:00:00 +Chicago New York 2010-10-21 142.0 361 2010-10-21 07:00:00 +Chicago New York 2010-10-22 2.0 361 2010-10-22 07:00:00 +Chicago New York 2010-10-24 10.0 361 2010-10-24 07:00:00 +Chicago New York 2010-10-25 31.0 361 2010-10-25 07:00:00 +Chicago New York 2010-10-26 12.0 361 2010-10-26 07:00:00 +Chicago New York 2010-10-27 148.0 361 2010-10-27 07:00:00 +Chicago New York 2010-10-28 2.0 361 2010-10-28 07:00:00 +Chicago New York 2010-10-29 -12.0 361 2010-10-29 07:00:00 +Chicago New York 2010-10-31 -25.0 361 2010-10-31 07:00:00 +Chicago New York 2010-10-23 3.0 384 2010-10-23 07:00:00 +Chicago New York 2010-10-30 -6.0 384 2010-10-30 07:00:00 +Chicago New York 2010-10-23 32.0 426 2010-10-23 07:00:00 +Chicago New York 2010-10-30 -10.0 426 2010-10-30 07:00:00 +Chicago New York 2010-10-23 1.0 650 2010-10-23 07:00:00 +Chicago New York 2010-10-30 -5.0 650 2010-10-30 07:00:00 +Chicago New York 2010-10-20 24.0 897 2010-10-20 07:00:00 +Chicago New York 2010-10-21 77.0 897 2010-10-21 07:00:00 +Chicago New York 2010-10-22 24.0 897 2010-10-22 07:00:00 +Chicago New York 2010-10-24 113.0 897 2010-10-24 07:00:00 +Chicago New York 2010-10-25 -1.0 897 2010-10-25 07:00:00 +Chicago New York 2010-10-26 0.0 897 2010-10-26 07:00:00 +Chicago New York 2010-10-27 -11.0 897 2010-10-27 07:00:00 +PREHOOK: query: select fl_time, count(*) from flights_tiny_orc_partitioned_timestamp group by fl_time +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 +#### A masked pattern was here #### +POSTHOOK: query: select fl_time, count(*) from flights_tiny_orc_partitioned_timestamp group by fl_time +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 +#### A masked pattern was here #### +2010-10-20 07:00:00 11 +2010-10-21 07:00:00 12 +2010-10-22 07:00:00 11 +2010-10-23 07:00:00 12 +2010-10-24 07:00:00 12 +2010-10-25 07:00:00 12 +2010-10-26 07:00:00 13 +2010-10-27 07:00:00 11 +2010-10-28 07:00:00 12 +2010-10-29 07:00:00 12 +2010-10-30 07:00:00 11 +2010-10-31 07:00:00 8 +PREHOOK: query: explain +select * from flights_tiny_orc_partitioned_timestamp +PREHOOK: type: QUERY +POSTHOOK: query: explain +select * from flights_tiny_orc_partitioned_timestamp +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: flights_tiny_orc_partitioned_timestamp + Statistics: Num rows: 137 Data size: 33968 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), dest_city_name (type: string), fl_date (type: date), arr_delay (type: float), fl_num (type: int), fl_time (type: timestamp) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 137 Data size: 33968 Basic stats: COMPLETE Column stats: NONE + ListSink + +PREHOOK: query: select * from flights_tiny_orc_partitioned_timestamp +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 +#### A masked pattern was here #### +POSTHOOK: query: select * from flights_tiny_orc_partitioned_timestamp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 +#### A masked pattern was here #### +Baltimore New York 2010-10-20 -30.0 1064 2010-10-20 07:00:00 +Baltimore New York 2010-10-20 23.0 1142 2010-10-20 07:00:00 +Baltimore New York 2010-10-20 6.0 1599 2010-10-20 07:00:00 +Chicago New York 2010-10-20 42.0 361 2010-10-20 07:00:00 +Chicago New York 2010-10-20 24.0 897 2010-10-20 07:00:00 +Chicago New York 2010-10-20 15.0 1531 2010-10-20 07:00:00 +Chicago New York 2010-10-20 -6.0 1610 2010-10-20 07:00:00 +Chicago New York 2010-10-20 -2.0 3198 2010-10-20 07:00:00 +Cleveland New York 2010-10-20 -8.0 2630 2010-10-20 07:00:00 +Cleveland New York 2010-10-20 -15.0 3014 2010-10-20 07:00:00 +Washington New York 2010-10-20 -2.0 7291 2010-10-20 07:00:00 +Baltimore New York 2010-10-21 17.0 1064 2010-10-21 07:00:00 +Baltimore New York 2010-10-21 105.0 1142 2010-10-21 07:00:00 +Baltimore New York 2010-10-21 28.0 1599 2010-10-21 07:00:00 +Chicago New York 2010-10-21 142.0 361 2010-10-21 07:00:00 +Chicago New York 2010-10-21 77.0 897 2010-10-21 07:00:00 +Chicago New York 2010-10-21 53.0 1531 2010-10-21 07:00:00 +Chicago New York 2010-10-21 -5.0 1610 2010-10-21 07:00:00 +Chicago New York 2010-10-21 51.0 3198 2010-10-21 07:00:00 +Cleveland New York 2010-10-21 3.0 2630 2010-10-21 07:00:00 +Cleveland New York 2010-10-21 29.0 2646 2010-10-21 07:00:00 +Cleveland New York 2010-10-21 72.0 3014 2010-10-21 07:00:00 +Washington New York 2010-10-21 22.0 7291 2010-10-21 07:00:00 +Baltimore New York 2010-10-22 -12.0 1064 2010-10-22 07:00:00 +Baltimore New York 2010-10-22 54.0 1142 2010-10-22 07:00:00 +Baltimore New York 2010-10-22 18.0 1599 2010-10-22 07:00:00 +Chicago New York 2010-10-22 2.0 361 2010-10-22 07:00:00 +Chicago New York 2010-10-22 24.0 897 2010-10-22 07:00:00 +Chicago New York 2010-10-22 16.0 1531 2010-10-22 07:00:00 +Chicago New York 2010-10-22 -6.0 1610 2010-10-22 07:00:00 +Chicago New York 2010-10-22 -11.0 3198 2010-10-22 07:00:00 +Cleveland New York 2010-10-22 1.0 2630 2010-10-22 07:00:00 +Cleveland New York 2010-10-22 -25.0 2646 2010-10-22 07:00:00 +Cleveland New York 2010-10-22 -3.0 3014 2010-10-22 07:00:00 +Baltimore New York 2010-10-23 18.0 272 2010-10-23 07:00:00 +Baltimore New York 2010-10-23 -10.0 1805 2010-10-23 07:00:00 +Baltimore New York 2010-10-23 6.0 3171 2010-10-23 07:00:00 +Chicago New York 2010-10-23 3.0 384 2010-10-23 07:00:00 +Chicago New York 2010-10-23 32.0 426 2010-10-23 07:00:00 +Chicago New York 2010-10-23 1.0 650 2010-10-23 07:00:00 +Chicago New York 2010-10-23 11.0 3085 2010-10-23 07:00:00 +Cleveland New York 2010-10-23 -21.0 2932 2010-10-23 07:00:00 +Washington New York 2010-10-23 -25.0 5832 2010-10-23 07:00:00 +Washington New York 2010-10-23 -21.0 5904 2010-10-23 07:00:00 +Washington New York 2010-10-23 -18.0 5917 2010-10-23 07:00:00 +Washington New York 2010-10-23 -16.0 7274 2010-10-23 07:00:00 +Baltimore New York 2010-10-24 12.0 1599 2010-10-24 07:00:00 +Baltimore New York 2010-10-24 20.0 2571 2010-10-24 07:00:00 +Chicago New York 2010-10-24 10.0 361 2010-10-24 07:00:00 +Chicago New York 2010-10-24 113.0 897 2010-10-24 07:00:00 +Chicago New York 2010-10-24 -5.0 1531 2010-10-24 07:00:00 +Chicago New York 2010-10-24 -17.0 1610 2010-10-24 07:00:00 +Chicago New York 2010-10-24 -3.0 3198 2010-10-24 07:00:00 +Cleveland New York 2010-10-24 5.0 2254 2010-10-24 07:00:00 +Cleveland New York 2010-10-24 -11.0 2630 2010-10-24 07:00:00 +Cleveland New York 2010-10-24 -20.0 2646 2010-10-24 07:00:00 +Cleveland New York 2010-10-24 -9.0 3014 2010-10-24 07:00:00 +Washington New York 2010-10-24 -26.0 7282 2010-10-24 07:00:00 +Baltimore New York 2010-10-25 -25.0 1064 2010-10-25 07:00:00 +Baltimore New York 2010-10-25 92.0 1142 2010-10-25 07:00:00 +Baltimore New York 2010-10-25 106.0 1599 2010-10-25 07:00:00 +Chicago New York 2010-10-25 31.0 361 2010-10-25 07:00:00 +Chicago New York 2010-10-25 -1.0 897 2010-10-25 07:00:00 +Chicago New York 2010-10-25 43.0 1531 2010-10-25 07:00:00 +Chicago New York 2010-10-25 6.0 1610 2010-10-25 07:00:00 +Chicago New York 2010-10-25 -16.0 3198 2010-10-25 07:00:00 +Cleveland New York 2010-10-25 -4.0 2630 2010-10-25 07:00:00 +Cleveland New York 2010-10-25 81.0 2646 2010-10-25 07:00:00 +Cleveland New York 2010-10-25 42.0 3014 2010-10-25 07:00:00 +Washington New York 2010-10-25 9.0 7291 2010-10-25 07:00:00 +Baltimore New York 2010-10-26 -22.0 1064 2010-10-26 07:00:00 +Baltimore New York 2010-10-26 123.0 1142 2010-10-26 07:00:00 +Baltimore New York 2010-10-26 90.0 1599 2010-10-26 07:00:00 +Chicago New York 2010-10-26 12.0 361 2010-10-26 07:00:00 +Chicago New York 2010-10-26 0.0 897 2010-10-26 07:00:00 +Chicago New York 2010-10-26 29.0 1531 2010-10-26 07:00:00 +Chicago New York 2010-10-26 -17.0 1610 2010-10-26 07:00:00 +Chicago New York 2010-10-26 6.0 3198 2010-10-26 07:00:00 +Cleveland New York 2010-10-26 4.0 2630 2010-10-26 07:00:00 +Cleveland New York 2010-10-26 -27.0 2646 2010-10-26 07:00:00 +Cleveland New York 2010-10-26 -11.0 2662 2010-10-26 07:00:00 +Cleveland New York 2010-10-26 13.0 3014 2010-10-26 07:00:00 +Washington New York 2010-10-26 4.0 7291 2010-10-26 07:00:00 +Baltimore New York 2010-10-27 -18.0 1064 2010-10-27 07:00:00 +Baltimore New York 2010-10-27 49.0 1142 2010-10-27 07:00:00 +Baltimore New York 2010-10-27 92.0 1599 2010-10-27 07:00:00 +Chicago New York 2010-10-27 148.0 361 2010-10-27 07:00:00 +Chicago New York 2010-10-27 -11.0 897 2010-10-27 07:00:00 +Chicago New York 2010-10-27 70.0 1531 2010-10-27 07:00:00 +Chicago New York 2010-10-27 8.0 1610 2010-10-27 07:00:00 +Chicago New York 2010-10-27 21.0 3198 2010-10-27 07:00:00 +Cleveland New York 2010-10-27 16.0 2630 2010-10-27 07:00:00 +Cleveland New York 2010-10-27 27.0 3014 2010-10-27 07:00:00 +Washington New York 2010-10-27 26.0 7291 2010-10-27 07:00:00 +Baltimore New York 2010-10-28 -4.0 1064 2010-10-28 07:00:00 +Baltimore New York 2010-10-28 -14.0 1142 2010-10-28 07:00:00 +Baltimore New York 2010-10-28 -14.0 1599 2010-10-28 07:00:00 +Chicago New York 2010-10-28 2.0 361 2010-10-28 07:00:00 +Chicago New York 2010-10-28 2.0 897 2010-10-28 07:00:00 +Chicago New York 2010-10-28 -11.0 1531 2010-10-28 07:00:00 +Chicago New York 2010-10-28 3.0 1610 2010-10-28 07:00:00 +Chicago New York 2010-10-28 -18.0 3198 2010-10-28 07:00:00 +Cleveland New York 2010-10-28 3.0 2630 2010-10-28 07:00:00 +Cleveland New York 2010-10-28 -6.0 2646 2010-10-28 07:00:00 +Cleveland New York 2010-10-28 1.0 3014 2010-10-28 07:00:00 +Washington New York 2010-10-28 45.0 7291 2010-10-28 07:00:00 +Baltimore New York 2010-10-29 -24.0 1064 2010-10-29 07:00:00 +Baltimore New York 2010-10-29 21.0 1142 2010-10-29 07:00:00 +Baltimore New York 2010-10-29 -2.0 1599 2010-10-29 07:00:00 +Chicago New York 2010-10-29 -12.0 361 2010-10-29 07:00:00 +Chicago New York 2010-10-29 -11.0 897 2010-10-29 07:00:00 +Chicago New York 2010-10-29 15.0 1531 2010-10-29 07:00:00 +Chicago New York 2010-10-29 -18.0 1610 2010-10-29 07:00:00 +Chicago New York 2010-10-29 -4.0 3198 2010-10-29 07:00:00 +Cleveland New York 2010-10-29 -4.0 2630 2010-10-29 07:00:00 +Cleveland New York 2010-10-29 -19.0 2646 2010-10-29 07:00:00 +Cleveland New York 2010-10-29 -12.0 3014 2010-10-29 07:00:00 +Washington New York 2010-10-29 1.0 7291 2010-10-29 07:00:00 +Baltimore New York 2010-10-30 14.0 272 2010-10-30 07:00:00 +Baltimore New York 2010-10-30 -1.0 1805 2010-10-30 07:00:00 +Baltimore New York 2010-10-30 5.0 3171 2010-10-30 07:00:00 +Chicago New York 2010-10-30 -6.0 384 2010-10-30 07:00:00 +Chicago New York 2010-10-30 -10.0 426 2010-10-30 07:00:00 +Chicago New York 2010-10-30 -5.0 650 2010-10-30 07:00:00 +Chicago New York 2010-10-30 -5.0 3085 2010-10-30 07:00:00 +Cleveland New York 2010-10-30 -23.0 2018 2010-10-30 07:00:00 +Cleveland New York 2010-10-30 -12.0 2932 2010-10-30 07:00:00 +Washington New York 2010-10-30 -27.0 5904 2010-10-30 07:00:00 +Washington New York 2010-10-30 -16.0 5917 2010-10-30 07:00:00 +Baltimore New York 2010-10-31 -1.0 1599 2010-10-31 07:00:00 +Baltimore New York 2010-10-31 -14.0 2571 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -25.0 361 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -18.0 897 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -4.0 1531 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -22.0 1610 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -15.0 3198 2010-10-31 07:00:00 +Washington New York 2010-10-31 -18.0 7282 2010-10-31 07:00:00 +PREHOOK: query: explain +select * from flights_tiny_orc_partitioned_timestamp sort by fl_num, fl_time limit 25 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select * from flights_tiny_orc_partitioned_timestamp sort by fl_num, fl_time limit 25 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: flights_tiny_orc_partitioned_timestamp + Statistics: Num rows: 137 Data size: 33968 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), dest_city_name (type: string), fl_date (type: date), arr_delay (type: float), fl_num (type: int), fl_time (type: timestamp) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 137 Data size: 33968 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: int), _col5 (type: timestamp) + sort order: ++ + Statistics: Num rows: 137 Data size: 33968 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: date), _col3 (type: float) + Execution mode: vectorized + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: date), VALUE._col3 (type: float), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: timestamp) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 137 Data size: 33968 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 25 + Statistics: Num rows: 25 Data size: 6175 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col4 (type: int), _col5 (type: timestamp) + sort order: ++ + Statistics: Num rows: 25 Data size: 6175 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: date), _col3 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: date), VALUE._col3 (type: float), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: timestamp) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 25 Data size: 6175 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 25 + Statistics: Num rows: 25 Data size: 6175 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 25 Data size: 6175 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: 25 + Processor Tree: + ListSink + +PREHOOK: query: select * from flights_tiny_orc_partitioned_timestamp sort by fl_num, fl_time limit 25 +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 +#### A masked pattern was here #### +POSTHOOK: query: select * from flights_tiny_orc_partitioned_timestamp sort by fl_num, fl_time limit 25 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 +#### A masked pattern was here #### +Baltimore New York 2010-10-23 18.0 272 2010-10-23 07:00:00 +Baltimore New York 2010-10-30 14.0 272 2010-10-30 07:00:00 +Chicago New York 2010-10-20 42.0 361 2010-10-20 07:00:00 +Chicago New York 2010-10-21 142.0 361 2010-10-21 07:00:00 +Chicago New York 2010-10-22 2.0 361 2010-10-22 07:00:00 +Chicago New York 2010-10-24 10.0 361 2010-10-24 07:00:00 +Chicago New York 2010-10-25 31.0 361 2010-10-25 07:00:00 +Chicago New York 2010-10-26 12.0 361 2010-10-26 07:00:00 +Chicago New York 2010-10-27 148.0 361 2010-10-27 07:00:00 +Chicago New York 2010-10-28 2.0 361 2010-10-28 07:00:00 +Chicago New York 2010-10-29 -12.0 361 2010-10-29 07:00:00 +Chicago New York 2010-10-31 -25.0 361 2010-10-31 07:00:00 +Chicago New York 2010-10-23 3.0 384 2010-10-23 07:00:00 +Chicago New York 2010-10-30 -6.0 384 2010-10-30 07:00:00 +Chicago New York 2010-10-23 32.0 426 2010-10-23 07:00:00 +Chicago New York 2010-10-30 -10.0 426 2010-10-30 07:00:00 +Chicago New York 2010-10-23 1.0 650 2010-10-23 07:00:00 +Chicago New York 2010-10-30 -5.0 650 2010-10-30 07:00:00 +Chicago New York 2010-10-20 24.0 897 2010-10-20 07:00:00 +Chicago New York 2010-10-21 77.0 897 2010-10-21 07:00:00 +Chicago New York 2010-10-22 24.0 897 2010-10-22 07:00:00 +Chicago New York 2010-10-24 113.0 897 2010-10-24 07:00:00 +Chicago New York 2010-10-25 -1.0 897 2010-10-25 07:00:00 +Chicago New York 2010-10-26 0.0 897 2010-10-26 07:00:00 +Chicago New York 2010-10-27 -11.0 897 2010-10-27 07:00:00 +PREHOOK: query: explain +select fl_time, count(*) from flights_tiny_orc_partitioned_timestamp group by fl_time +PREHOOK: type: QUERY +POSTHOOK: query: explain +select fl_time, count(*) from flights_tiny_orc_partitioned_timestamp group by fl_time +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: flights_tiny_orc_partitioned_timestamp + Statistics: Num rows: 137 Data size: 33968 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: fl_time (type: timestamp) + outputColumnNames: fl_time + Statistics: Num rows: 137 Data size: 33968 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: fl_time (type: timestamp) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 137 Data size: 33968 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: timestamp) + sort order: + + Map-reduce partition columns: _col0 (type: timestamp) + Statistics: Num rows: 137 Data size: 33968 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: timestamp) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 68 Data size: 16860 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: timestamp), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 68 Data size: 16860 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 68 Data size: 16860 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select fl_time, count(*) from flights_tiny_orc_partitioned_timestamp group by fl_time +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 +PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 +#### A masked pattern was here #### +POSTHOOK: query: select fl_time, count(*) from flights_tiny_orc_partitioned_timestamp group by fl_time +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 +#### A masked pattern was here #### +2010-10-20 07:00:00 11 +2010-10-21 07:00:00 12 +2010-10-22 07:00:00 11 +2010-10-23 07:00:00 12 +2010-10-24 07:00:00 12 +2010-10-25 07:00:00 12 +2010-10-26 07:00:00 13 +2010-10-27 07:00:00 11 +2010-10-28 07:00:00 12 +2010-10-29 07:00:00 12 +2010-10-30 07:00:00 11 +2010-10-31 07:00:00 8 diff --git a/ql/src/test/results/clientpositive/vector_reduce_groupby_decimal.q.out b/ql/src/test/results/clientpositive/vector_reduce_groupby_decimal.q.out new file mode 100644 index 0000000..2343508 --- /dev/null +++ b/ql/src/test/results/clientpositive/vector_reduce_groupby_decimal.q.out @@ -0,0 +1,211 @@ +PREHOOK: query: CREATE TABLE decimal_test STORED AS ORC AS SELECT cint, cdouble, CAST (((cdouble*22.1)/37) AS DECIMAL(20,10)) AS cdecimal1, CAST (((cdouble*9.3)/13) AS DECIMAL(23,14)) AS cdecimal2 FROM alltypesorc +WHERE cint is not null and cdouble is not null +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: database:default +PREHOOK: Output: default@decimal_test +POSTHOOK: query: CREATE TABLE decimal_test STORED AS ORC AS SELECT cint, cdouble, CAST (((cdouble*22.1)/37) AS DECIMAL(20,10)) AS cdecimal1, CAST (((cdouble*9.3)/13) AS DECIMAL(23,14)) AS cdecimal2 FROM alltypesorc +WHERE cint is not null and cdouble is not null +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: database:default +POSTHOOK: Output: default@decimal_test +PREHOOK: query: EXPLAIN +SELECT cint, cdouble, cdecimal1, cdecimal2, min(cdecimal1) as min_decimal1 FROM decimal_test +WHERE cdecimal1 is not null and cdecimal2 is not null +GROUP BY cint, cdouble, cdecimal1, cdecimal2 +ORDER BY cint, cdouble, cdecimal1, cdecimal2 +LIMIT 50 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT cint, cdouble, cdecimal1, cdecimal2, min(cdecimal1) as min_decimal1 FROM decimal_test +WHERE cdecimal1 is not null and cdecimal2 is not null +GROUP BY cint, cdouble, cdecimal1, cdecimal2 +ORDER BY cint, cdouble, cdecimal1, cdecimal2 +LIMIT 50 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: decimal_test + Statistics: Num rows: 6102 Data size: 1440072 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (cdecimal1 is not null and cdecimal2 is not null) (type: boolean) + Statistics: Num rows: 1526 Data size: 360136 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cint (type: int), cdouble (type: double), cdecimal1 (type: decimal(20,10)), cdecimal2 (type: decimal(23,14)) + outputColumnNames: cint, cdouble, cdecimal1, cdecimal2 + Statistics: Num rows: 1526 Data size: 360136 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: min(cdecimal1) + keys: cint (type: int), cdouble (type: double), cdecimal1 (type: decimal(20,10)), cdecimal2 (type: decimal(23,14)) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1526 Data size: 360136 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: double), _col2 (type: decimal(20,10)), _col3 (type: decimal(23,14)) + sort order: ++++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: double), _col2 (type: decimal(20,10)), _col3 (type: decimal(23,14)) + Statistics: Num rows: 1526 Data size: 360136 Basic stats: COMPLETE Column stats: NONE + value expressions: _col4 (type: decimal(20,10)) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: min(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: double), KEY._col2 (type: decimal(20,10)), KEY._col3 (type: decimal(23,14)) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 763 Data size: 180068 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: double), _col2 (type: decimal(20,10)), _col3 (type: decimal(23,14)), _col4 (type: decimal(20,10)) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 763 Data size: 180068 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: double), _col2 (type: decimal(20,10)), _col3 (type: decimal(23,14)) + sort order: ++++ + Statistics: Num rows: 763 Data size: 180068 Basic stats: COMPLETE Column stats: NONE + value expressions: _col4 (type: decimal(20,10)) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: double), KEY.reducesinkkey2 (type: decimal(20,10)), KEY.reducesinkkey3 (type: decimal(23,14)), VALUE._col0 (type: decimal(20,10)) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 763 Data size: 180068 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 50 + Statistics: Num rows: 50 Data size: 11800 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 50 Data size: 11800 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: 50 + Processor Tree: + ListSink + +PREHOOK: query: SELECT cint, cdouble, cdecimal1, cdecimal2, min(cdecimal1) as min_decimal1 FROM decimal_test +WHERE cdecimal1 is not null and cdecimal2 is not null +GROUP BY cint, cdouble, cdecimal1, cdecimal2 +ORDER BY cint, cdouble, cdecimal1, cdecimal2 +LIMIT 50 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_test +#### A masked pattern was here #### +POSTHOOK: query: SELECT cint, cdouble, cdecimal1, cdecimal2, min(cdecimal1) as min_decimal1 FROM decimal_test +WHERE cdecimal1 is not null and cdecimal2 is not null +GROUP BY cint, cdouble, cdecimal1, cdecimal2 +ORDER BY cint, cdouble, cdecimal1, cdecimal2 +LIMIT 50 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_test +#### A masked pattern was here #### +-1073051226 -7382.0 -4409.2486486486 -5280.969230769231 -4409.2486486486 +-1072081801 8373.0 5001.1702702703 5989.915384615385 5001.1702702703 +-1072076362 -5470.0 -3267.2162162162 -3913.1538461538466 -3267.2162162162 +-1070883071 -741.0 -442.5972972973 -530.1 -442.5972972973 +-1070551679 -947.0 -565.6405405405 -677.4692307692308 -565.6405405405 +-1069512165 11417.0 6819.3432432432 8167.546153846154 6819.3432432432 +-1069109166 8390.0 5011.3243243243 6002.076923076923 5011.3243243243 +-1068623584 -14005.0 -8365.1486486486 -10018.961538461539 -8365.1486486486 +-1067386090 -3977.0 -2375.4513513514 -2845.084615384616 -2375.4513513514 +-1066922682 -9987.0 -5965.2081081081 -7144.546153846154 -5965.2081081081 +-1066226047 -9439.0 -5637.8891891892 -6752.515384615385 -5637.8891891892 +-1065117869 2538.0 1515.9405405405 1815.646153846154 1515.9405405405 +-1064949302 6454.0 3854.9567567568 4617.092307692308 3854.9567567568 +-1063498122 -11480.0 -6856.9729729730 -8212.615384615387 -6856.9729729730 +-1062973443 10541.0 6296.1108108108 7540.869230769231 6296.1108108108 +-1061614989 -4234.0 -2528.9567567568 -3028.938461538462 -2528.9567567568 +-1061057428 -1085.0 -648.0675675676 -776.1923076923077 -648.0675675676 +-1059941909 8782.0 5245.4648648649 6282.507692307693 5245.4648648649 +-1059338191 7322.0 4373.4108108108 5238.046153846154 4373.4108108108 +-1059047258 12452.0 7437.5459459459 8907.969230769231 7437.5459459459 +-1056684111 13991.0 8356.7864864865 10008.946153846155 8356.7864864865 +-1055945837 13690.0 8177.0 9793.615384615387 8177.0 +-1055669248 2570.0 1535.0540540541 1838.538461538462 1535.0540540541 +-1055316250 -14990.0 -8953.4864864865 -10723.615384615385 -8953.4864864865 +-1053385587 14504.0 8663.2 10375.938461538462 8663.2 +-1053238077 -3704.0 -2212.3891891892 -2649.784615384616 -2212.3891891892 +-1052745800 -12404.0 -7408.8756756757 -8873.630769230771 -7408.8756756757 +-1052322972 -7433.0 -4439.7108108108 -5317.453846153847 -4439.7108108108 +-1050684541 -8261.0 -4934.2729729730 -5909.792307692308 -4934.2729729730 +-1050657303 -6999.0 -4180.4837837838 -5006.976923076923 -4180.4837837838 +-1050165799 8634.0 5157.0648648649 6176.63076923077 5157.0648648649 +-1048934049 -524.0 -312.9837837838 -374.86153846153854 -312.9837837838 +-1046399794 4130.0 2466.8378378378 2954.5384615384614 2466.8378378378 +-1045867222 -8034.0 -4798.6864864865 -5747.400000000001 -4798.6864864865 +-1045196363 -5039.0 -3009.7810810811 -3604.823076923077 -3009.7810810811 +-1045181724 -5706.0 -3408.1783783784 -4081.9846153846156 -3408.1783783784 +-1045087657 -5865.0 -3503.1486486486 -4195.7307692307695 -3503.1486486486 +-1044207190 5381.0 3214.0567567568 3849.4846153846156 3214.0567567568 +-1044093617 -3422.0 -2043.9513513514 -2448.046153846154 -2043.9513513514 +-1043573508 16216.0 9685.7729729730 11600.676923076924 9685.7729729730 +-1043132597 12302.0 7347.9513513514 8800.66153846154 7347.9513513514 +-1043082182 9180.0 5483.1891891892 6567.2307692307695 5483.1891891892 +-1042805968 5133.0 3065.9270270270 3672.0692307692307 3065.9270270270 +-1042712895 9296.0 5552.4756756757 6650.215384615385 5552.4756756757 +-1042396242 9583.0 5723.9000000000 6855.53076923077 5723.9000000000 +-1041734429 -836.0 -499.3405405405 -598.0615384615385 -499.3405405405 +-1041391389 -12970.0 -7746.9459459459 -9278.538461538463 -7746.9459459459 +-1041252354 756.0 451.5567567568 540.8307692307692 451.5567567568 +-1039776293 13704.0 8185.3621621622 9803.630769230771 8185.3621621622 +-1039762548 -3802.0 -2270.9243243243 -2719.8923076923083 -2270.9243243243 +PREHOOK: query: SELECT sum(hash(*)) + FROM (SELECT cint, cdouble, cdecimal1, cdecimal2, min(cdecimal1) as min_decimal1 FROM decimal_test + WHERE cdecimal1 is not null and cdecimal2 is not null + GROUP BY cint, cdouble, cdecimal1, cdecimal2 + ORDER BY cint, cdouble, cdecimal1, cdecimal2 + LIMIT 50) as q +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_test +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(*)) + FROM (SELECT cint, cdouble, cdecimal1, cdecimal2, min(cdecimal1) as min_decimal1 FROM decimal_test + WHERE cdecimal1 is not null and cdecimal2 is not null + GROUP BY cint, cdouble, cdecimal1, cdecimal2 + ORDER BY cint, cdouble, cdecimal1, cdecimal2 + LIMIT 50) as q +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_test +#### A masked pattern was here #### +12703057972 +PREHOOK: query: SELECT sum(hash(*)) + FROM (SELECT cint, cdouble, cdecimal1, cdecimal2, min(cdecimal1) as min_decimal1 FROM decimal_test + WHERE cdecimal1 is not null and cdecimal2 is not null + GROUP BY cint, cdouble, cdecimal1, cdecimal2 + ORDER BY cint, cdouble, cdecimal1, cdecimal2 + LIMIT 50) as q +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_test +#### A masked pattern was here #### +POSTHOOK: query: SELECT sum(hash(*)) + FROM (SELECT cint, cdouble, cdecimal1, cdecimal2, min(cdecimal1) as min_decimal1 FROM decimal_test + WHERE cdecimal1 is not null and cdecimal2 is not null + GROUP BY cint, cdouble, cdecimal1, cdecimal2 + ORDER BY cint, cdouble, cdecimal1, cdecimal2 + LIMIT 50) as q +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_test +#### A masked pattern was here #### +12703057972 diff --git a/ql/src/test/results/clientpositive/windowing.q.out b/ql/src/test/results/clientpositive/windowing.q.out index 31d0511..a16e73a 100644 --- a/ql/src/test/results/clientpositive/windowing.q.out +++ b/ql/src/test/results/clientpositive/windowing.q.out @@ -2362,3 +2362,15 @@ POSTHOOK: Input: default@part 1602.59 1549.8900000000003 4649.670000000001 1414.42 1523.5400000000004 3047.080000000001 1632.66 1632.6600000000008 1632.6600000000008 +PREHOOK: query: -- 47. empty partition +select sum(p_size) over (partition by p_mfgr ) +from part where p_mfgr = 'm1' +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 47. empty partition +select sum(p_size) over (partition by p_mfgr ) +from part where p_mfgr = 'm1' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### diff --git a/ql/src/test/results/compiler/plan/cast1.q.xml b/ql/src/test/results/compiler/plan/cast1.q.xml index e135567..1bba170 100644 --- a/ql/src/test/results/compiler/plan/cast1.q.xml +++ b/ql/src/test/results/compiler/plan/cast1.q.xml @@ -379,6 +379,9 @@ _col6 + + UDFToInteger(true) + @@ -390,6 +393,9 @@ _col5 + + UDFToBoolean(1) + @@ -401,6 +407,9 @@ _col4 + + (3 + UDFToInteger(2.0)) + @@ -412,6 +421,9 @@ _col3 + + (3.0 + 2.0) + @@ -423,6 +435,9 @@ _col2 + + (3 + 2.0) + @@ -434,6 +449,9 @@ _col1 + + (3.0 + 2) + @@ -445,6 +463,9 @@ _col0 + + (3 + 2) + diff --git a/ql/src/test/results/compiler/plan/groupby1.q.xml b/ql/src/test/results/compiler/plan/groupby1.q.xml index 4f2e132..3b7e69b 100755 --- a/ql/src/test/results/compiler/plan/groupby1.q.xml +++ b/ql/src/test/results/compiler/plan/groupby1.q.xml @@ -429,6 +429,14 @@ + + + + + UNSET + + + -1 diff --git a/ql/src/test/results/compiler/plan/groupby2.q.xml b/ql/src/test/results/compiler/plan/groupby2.q.xml index fe050e2..3a1164c 100755 --- a/ql/src/test/results/compiler/plan/groupby2.q.xml +++ b/ql/src/test/results/compiler/plan/groupby2.q.xml @@ -309,6 +309,14 @@ + + + + + UNSET + + + -1 diff --git a/ql/src/test/results/compiler/plan/groupby3.q.xml b/ql/src/test/results/compiler/plan/groupby3.q.xml index 2246a7b..b33a2c6 100644 --- a/ql/src/test/results/compiler/plan/groupby3.q.xml +++ b/ql/src/test/results/compiler/plan/groupby3.q.xml @@ -364,6 +364,14 @@ + + + + + UNSET + + + -1 diff --git a/ql/src/test/results/compiler/plan/groupby4.q.xml b/ql/src/test/results/compiler/plan/groupby4.q.xml index 909f6dd..a61b055 100644 --- a/ql/src/test/results/compiler/plan/groupby4.q.xml +++ b/ql/src/test/results/compiler/plan/groupby4.q.xml @@ -259,6 +259,14 @@ + + + + + UNSET + + + -1 diff --git a/ql/src/test/results/compiler/plan/groupby5.q.xml b/ql/src/test/results/compiler/plan/groupby5.q.xml index 1106075..2946c67 100644 --- a/ql/src/test/results/compiler/plan/groupby5.q.xml +++ b/ql/src/test/results/compiler/plan/groupby5.q.xml @@ -281,6 +281,14 @@ + + + + + UNSET + + + -1 diff --git a/ql/src/test/results/compiler/plan/groupby6.q.xml b/ql/src/test/results/compiler/plan/groupby6.q.xml index 86c00eb..08bf86f 100644 --- a/ql/src/test/results/compiler/plan/groupby6.q.xml +++ b/ql/src/test/results/compiler/plan/groupby6.q.xml @@ -259,6 +259,14 @@ + + + + + UNSET + + + -1 diff --git a/ql/src/test/results/compiler/plan/input20.q.xml b/ql/src/test/results/compiler/plan/input20.q.xml index 08d45a6..845367d 100644 --- a/ql/src/test/results/compiler/plan/input20.q.xml +++ b/ql/src/test/results/compiler/plan/input20.q.xml @@ -292,6 +292,14 @@ + + + + + UNSET + + + -1 diff --git a/ql/src/test/results/compiler/plan/input4.q.xml b/ql/src/test/results/compiler/plan/input4.q.xml index 2daf426..f9d5340 100755 --- a/ql/src/test/results/compiler/plan/input4.q.xml +++ b/ql/src/test/results/compiler/plan/input4.q.xml @@ -443,6 +443,14 @@ + + + + + UNSET + + + -1 diff --git a/ql/src/test/results/compiler/plan/input5.q.xml b/ql/src/test/results/compiler/plan/input5.q.xml index 22afeb1..a211641 100644 --- a/ql/src/test/results/compiler/plan/input5.q.xml +++ b/ql/src/test/results/compiler/plan/input5.q.xml @@ -448,6 +448,14 @@ + + + + + UNSET + + + -1 diff --git a/ql/src/test/results/compiler/plan/input_part1.q.xml b/ql/src/test/results/compiler/plan/input_part1.q.xml index d87834d..3640102 100644 --- a/ql/src/test/results/compiler/plan/input_part1.q.xml +++ b/ql/src/test/results/compiler/plan/input_part1.q.xml @@ -351,41 +351,35 @@ _col3 - - + + ds - - true - - - srcpart - + + 2008-04-08 + _col2 - - + + hr - - true - - - srcpart - + + 12 + _col1 - + value @@ -399,7 +393,7 @@ _col0 - + key @@ -418,16 +412,16 @@ - + - + - + - + @@ -634,38 +628,6 @@ - - - - ds - - - srcpart - - - - - - string - - - - - - - hr - - - srcpart - - - - - - string - - - @@ -713,12 +675,6 @@ value - - ds - - - hr - @@ -731,10 +687,42 @@ - + + + ds + + + ds + + + srcpart + + + + + + string + + - + + + hr + + + hr + + + srcpart + + + + + + string + + diff --git a/ql/src/test/results/compiler/plan/join1.q.xml b/ql/src/test/results/compiler/plan/join1.q.xml index a78972f..9a1c202 100644 --- a/ql/src/test/results/compiler/plan/join1.q.xml +++ b/ql/src/test/results/compiler/plan/join1.q.xml @@ -487,6 +487,14 @@ + + + + + UNSET + + + 1 @@ -890,6 +898,12 @@ + + + + + + diff --git a/ql/src/test/results/compiler/plan/join2.q.xml b/ql/src/test/results/compiler/plan/join2.q.xml index 2ee352c..28a7d8f 100644 --- a/ql/src/test/results/compiler/plan/join2.q.xml +++ b/ql/src/test/results/compiler/plan/join2.q.xml @@ -465,6 +465,14 @@ + + + + + UNSET + + + @@ -741,6 +749,12 @@ + + + + + + 1 @@ -1944,6 +1958,12 @@ + + + + + + 1 @@ -2273,6 +2293,12 @@ + + + + + + diff --git a/ql/src/test/results/compiler/plan/join3.q.xml b/ql/src/test/results/compiler/plan/join3.q.xml index a6e403e..93914da 100644 --- a/ql/src/test/results/compiler/plan/join3.q.xml +++ b/ql/src/test/results/compiler/plan/join3.q.xml @@ -534,6 +534,14 @@ + + + + + UNSET + + + 1 @@ -920,6 +928,12 @@ + + + + + + 2 @@ -1284,6 +1298,12 @@ + + + + + + diff --git a/ql/src/test/results/compiler/plan/join4.q.xml b/ql/src/test/results/compiler/plan/join4.q.xml index d3e9127..63fcd64 100644 --- a/ql/src/test/results/compiler/plan/join4.q.xml +++ b/ql/src/test/results/compiler/plan/join4.q.xml @@ -336,6 +336,14 @@ + + + + + UNSET + + + @@ -911,6 +919,12 @@ + + + + + + 1 diff --git a/ql/src/test/results/compiler/plan/join5.q.xml b/ql/src/test/results/compiler/plan/join5.q.xml index 8615beb..71487a6 100644 --- a/ql/src/test/results/compiler/plan/join5.q.xml +++ b/ql/src/test/results/compiler/plan/join5.q.xml @@ -336,6 +336,14 @@ + + + + + UNSET + + + @@ -911,6 +919,12 @@ + + + + + + 1 diff --git a/ql/src/test/results/compiler/plan/join6.q.xml b/ql/src/test/results/compiler/plan/join6.q.xml index 473d217..3ff2610 100644 --- a/ql/src/test/results/compiler/plan/join6.q.xml +++ b/ql/src/test/results/compiler/plan/join6.q.xml @@ -336,6 +336,14 @@ + + + + + UNSET + + + @@ -911,6 +919,12 @@ + + + + + + 1 diff --git a/ql/src/test/results/compiler/plan/join7.q.xml b/ql/src/test/results/compiler/plan/join7.q.xml index 5568a7a..279147a 100644 --- a/ql/src/test/results/compiler/plan/join7.q.xml +++ b/ql/src/test/results/compiler/plan/join7.q.xml @@ -401,6 +401,14 @@ + + + + + UNSET + + + @@ -976,6 +984,12 @@ + + + + + + 1 @@ -1515,6 +1529,12 @@ + + + + + + 2 diff --git a/ql/src/test/results/compiler/plan/join8.q.xml b/ql/src/test/results/compiler/plan/join8.q.xml index 7ef3d43..c9a2de4 100644 --- a/ql/src/test/results/compiler/plan/join8.q.xml +++ b/ql/src/test/results/compiler/plan/join8.q.xml @@ -336,6 +336,14 @@ + + + + + UNSET + + + @@ -952,6 +960,12 @@ + + + + + + 1 diff --git a/ql/src/test/results/compiler/plan/sample1.q.xml b/ql/src/test/results/compiler/plan/sample1.q.xml index 0541c16..0ab3b87 100644 --- a/ql/src/test/results/compiler/plan/sample1.q.xml +++ b/ql/src/test/results/compiler/plan/sample1.q.xml @@ -351,41 +351,35 @@ _col3 - - + + hr - - true - - - s - + + 11 + _col2 - - + + ds - - true - - - s - + + 2008-04-08 + _col1 - + value @@ -399,7 +393,7 @@ _col0 - + key @@ -418,16 +412,16 @@ - + - + - + - + @@ -727,38 +721,6 @@ - - - - ds - - - s - - - - - - string - - - - - - - hr - - - s - - - - - - string - - - @@ -806,12 +768,6 @@ value - - ds - - - hr - @@ -824,10 +780,42 @@ - + + + ds + + + ds + + + s + + + + + + string + + - + + + hr + + + hr + + + s + + + + + + string + + diff --git a/ql/src/test/results/compiler/plan/udf1.q.xml b/ql/src/test/results/compiler/plan/udf1.q.xml index 44988ac..a6c45b2 100644 --- a/ql/src/test/results/compiler/plan/udf1.q.xml +++ b/ql/src/test/results/compiler/plan/udf1.q.xml @@ -535,6 +535,9 @@ _col8 + + ('' rlike '.*') + @@ -546,6 +549,9 @@ _col7 + + ('ab' like 'a') + @@ -557,6 +563,9 @@ _col6 + + ('ab' like '_a%') + @@ -568,6 +577,9 @@ _col5 + + ('ab' like '\%\_') + @@ -579,6 +591,9 @@ _col4 + + ('%_' like '\%\_') + @@ -590,6 +605,9 @@ _col3 + + ('ab' like '%a_') + @@ -601,6 +619,9 @@ _col2 + + ('ab' like '%a%') + @@ -612,6 +633,9 @@ _col1 + + ('b' like '%a%') + @@ -623,6 +647,9 @@ _col9 + + ('a' rlike '[ab]') + @@ -634,6 +661,9 @@ _col13 + + regexp_replace('abc', 'b', 'c') + @@ -645,6 +675,9 @@ _col12 + + ('hadoop' rlike 'o*') + @@ -656,6 +689,9 @@ _col11 + + ('hadoop' rlike '[a-z]*') + @@ -667,6 +703,9 @@ _col10 + + ('' rlike '[ab]') + @@ -678,6 +717,9 @@ _col16 + + regexp_replace('hadoop', '(.)[a-z]*', '$1ive') + @@ -689,6 +731,9 @@ _col15 + + regexp_replace('abbbb', 'bb', 'b') + @@ -700,6 +745,9 @@ _col14 + + regexp_replace('abc', 'z', 'a') + @@ -711,6 +759,9 @@ _col0 + + ('a' like '%a%') + diff --git a/ql/src/test/results/compiler/plan/udf4.q.xml b/ql/src/test/results/compiler/plan/udf4.q.xml index 4ca78bf..1edf38a 100644 --- a/ql/src/test/results/compiler/plan/udf4.q.xml +++ b/ql/src/test/results/compiler/plan/udf4.q.xml @@ -548,6 +548,9 @@ _col8 + + sqrt(0.0) + @@ -563,6 +566,9 @@ _col6 + + sqrt(1.0) + @@ -574,6 +580,9 @@ _col5 + + floor((- 1.5)) + @@ -585,6 +594,9 @@ _col4 + + floor(1.5) + @@ -596,6 +608,9 @@ _col3 + + floor(1.0) + @@ -607,6 +622,9 @@ _col2 + + round((- 1.5)) + @@ -618,6 +636,9 @@ _col1 + + round(1.5) + @@ -629,6 +650,9 @@ _col9 + + ceil(1.0) + @@ -672,6 +696,9 @@ _col12 + + ceil(1.0) + @@ -683,6 +710,9 @@ _col11 + + ceil((- 1.5)) + @@ -694,6 +724,9 @@ _col10 + + ceil(1.5) + @@ -705,6 +738,9 @@ _col17 + + (1 + (- 2)) + @@ -716,6 +752,9 @@ _col16 + + (1 + 2) + @@ -727,6 +766,9 @@ _col15 + + (- 3) + @@ -749,6 +791,9 @@ _col0 + + round(1.0) + @@ -760,6 +805,9 @@ _col18 + + (~ 1) + diff --git a/ql/src/test/results/compiler/plan/udf6.q.xml b/ql/src/test/results/compiler/plan/udf6.q.xml index 24008df..c4c8980 100644 --- a/ql/src/test/results/compiler/plan/udf6.q.xml +++ b/ql/src/test/results/compiler/plan/udf6.q.xml @@ -338,6 +338,9 @@ _col0 + + concat('a', 'b') + diff --git a/ql/src/test/results/compiler/plan/udf_case.q.xml b/ql/src/test/results/compiler/plan/udf_case.q.xml index dc620cb..3518469 100644 --- a/ql/src/test/results/compiler/plan/udf_case.q.xml +++ b/ql/src/test/results/compiler/plan/udf_case.q.xml @@ -351,6 +351,9 @@ _col0 + + CASE (1) WHEN (1) THEN (2) WHEN (3) THEN (4) ELSE (5) END + diff --git a/ql/src/test/results/compiler/plan/udf_when.q.xml b/ql/src/test/results/compiler/plan/udf_when.q.xml index dc620cb..4a1d604 100644 --- a/ql/src/test/results/compiler/plan/udf_when.q.xml +++ b/ql/src/test/results/compiler/plan/udf_when.q.xml @@ -351,6 +351,9 @@ _col0 + + CASE WHEN ((1 = 1)) THEN (2) WHEN ((3 = 5)) THEN (4) ELSE (5) END + diff --git a/ql/src/test/scripts/transform_acid_grep.sh b/ql/src/test/scripts/transform_acid_grep.sh new file mode 100644 index 0000000..2b2001a --- /dev/null +++ b/ql/src/test/scripts/transform_acid_grep.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +env | grep hive.txn.valid.txns +echo a +exit 0 + diff --git a/serde/pom.xml b/serde/pom.xml index b7bc4f0..98e5506 100644 --- a/serde/pom.xml +++ b/serde/pom.xml @@ -19,7 +19,7 @@ org.apache.hive hive - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../pom.xml diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/AbstractSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/AbstractSerDe.java index 28cfe07..c5e78c5 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/AbstractSerDe.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/AbstractSerDe.java @@ -24,6 +24,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.io.Writable; +import javax.annotation.Nullable; + /** * Abstract class for implementing SerDe. The abstract class has been created, so that * new methods can be added in the underlying interface, SerDe, and only implementations @@ -31,6 +33,8 @@ */ public abstract class AbstractSerDe implements SerDe { + protected String configErrors; + /** * Initialize the SerDe. By default, this will use one set of properties, either the * table properties or the partition properties. If a SerDe needs access to both sets, @@ -54,13 +58,14 @@ public void initialize(Configuration configuration, Properties tableProperties, * Initialize the HiveSerializer. * * @param conf - * System properties + * System properties. Can be null in compile time * @param tbl * table properties * @throws SerDeException */ @Deprecated - public abstract void initialize(Configuration conf, Properties tbl) throws SerDeException; + public abstract void initialize(@Nullable Configuration conf, Properties tbl) + throws SerDeException; /** * Returns the Writable class that would be returned by the serialize method. @@ -101,4 +106,13 @@ public abstract Writable serialize(Object obj, ObjectInspector objInspector) * structure of the Object returned from deserialize(...). */ public abstract ObjectInspector getObjectInspector() throws SerDeException; + + /** + * Get the error messages during the Serde configuration + * + * @return The error messages in the configuration which are empty if no error occurred + */ + public String getConfigurationErrors() { + return configErrors == null ? "" : configErrors; + } } diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/MetadataTypedColumnsetSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/MetadataTypedColumnsetSerDe.java index 8cc752f..262c57f 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/MetadataTypedColumnsetSerDe.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/MetadataTypedColumnsetSerDe.java @@ -42,6 +42,11 @@ * MetadataTypedColumnsetSerDe. * */ +@SerDeSpec(schemaProps = { + serdeConstants.SERIALIZATION_FORMAT, + serdeConstants.SERIALIZATION_NULL_FORMAT, + serdeConstants.SERIALIZATION_LIB, + serdeConstants.SERIALIZATION_LAST_COLUMN_TAKES_REST }) public class MetadataTypedColumnsetSerDe extends AbstractSerDe { public static final Log LOG = LogFactory diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/OpenCSVSerde.java b/serde/src/java/org/apache/hadoop/hive/serde2/OpenCSVSerde.java index ac6c6dd..44b5ae7 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/OpenCSVSerde.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/OpenCSVSerde.java @@ -49,6 +49,9 @@ * quote("), and escape characters(\) are the same as the opencsv library. * */ +@SerDeSpec(schemaProps = { + serdeConstants.LIST_COLUMNS, + OpenCSVSerde.SEPARATORCHAR, OpenCSVSerde.QUOTECHAR, OpenCSVSerde.ESCAPECHAR}) public final class OpenCSVSerde extends AbstractSerDe { public static final Log LOG = LogFactory.getLog(OpenCSVSerde.class.getName()); @@ -69,7 +72,7 @@ public void initialize(final Configuration conf, final Properties tbl) throws SerDeException { final List columnNames = Arrays.asList(tbl.getProperty(serdeConstants.LIST_COLUMNS) - .split(",")); + .split(",")); numCols = columnNames.size(); diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/RegexSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/RegexSerDe.java index 0b8e0e5..38e8b77 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/RegexSerDe.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/RegexSerDe.java @@ -39,7 +39,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; @@ -70,10 +69,16 @@ * writableStringObjectInspector. We should switch to that when we have a UTF-8 * based Regex library. */ +@SerDeSpec(schemaProps = { + serdeConstants.LIST_COLUMNS, serdeConstants.LIST_COLUMN_TYPES, + RegexSerDe.INPUT_REGEX, RegexSerDe.INPUT_REGEX_CASE_SENSITIVE }) public class RegexSerDe extends AbstractSerDe { public static final Log LOG = LogFactory.getLog(RegexSerDe.class.getName()); + public static final String INPUT_REGEX = "input.regex"; + public static final String INPUT_REGEX_CASE_SENSITIVE = "input.regex.case.insensitive"; + int numColumns; String inputRegex; @@ -95,11 +100,11 @@ public void initialize(Configuration conf, Properties tbl) // We can get the table definition from tbl. // Read the configuration parameters - inputRegex = tbl.getProperty("input.regex"); + inputRegex = tbl.getProperty(INPUT_REGEX); String columnNameProperty = tbl.getProperty(serdeConstants.LIST_COLUMNS); String columnTypeProperty = tbl.getProperty(serdeConstants.LIST_COLUMN_TYPES); boolean inputRegexIgnoreCase = "true".equalsIgnoreCase(tbl - .getProperty("input.regex.case.insensitive")); + .getProperty(INPUT_REGEX_CASE_SENSITIVE)); // output format string is not supported anymore, warn user of deprecation if (null != tbl.getProperty("output.format.string")) { diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/SerDeSpec.java b/serde/src/java/org/apache/hadoop/hive/serde2/SerDeSpec.java new file mode 100644 index 0000000..3308004 --- /dev/null +++ b/serde/src/java/org/apache/hadoop/hive/serde2/SerDeSpec.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@Inherited +public @interface SerDeSpec { + // property names needed to keep internal structure of serde + String[] schemaProps(); +} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java b/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java index 274d468..c2649b4 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java @@ -497,7 +497,8 @@ public static boolean hasAnyNullObject(Object o, ObjectInspector oi) { * @return the overlayed properties */ public static Properties createOverlayedProperties(Properties tblProps, Properties partProps) { - Properties props = new Properties(tblProps); + Properties props = new Properties(); + props.putAll(tblProps); if (partProps != null) { props.putAll(partProps); } @@ -506,7 +507,8 @@ public static Properties createOverlayedProperties(Properties tblProps, Properti /** * Initializes a SerDe. - * @param serde + * @param deserializer + * @param conf * @param tblProps * @param partProps * @throws SerDeException @@ -516,6 +518,28 @@ public static void initializeSerDe(Deserializer deserializer, Configuration conf throws SerDeException { if (deserializer instanceof AbstractSerDe) { ((AbstractSerDe) deserializer).initialize(conf, tblProps, partProps); + String msg = ((AbstractSerDe) deserializer).getConfigurationErrors(); + if (msg != null && !msg.isEmpty()) { + throw new SerDeException(msg); + } + } else { + deserializer.initialize(conf, createOverlayedProperties(tblProps, partProps)); + } + } + + /** + * Initializes a SerDe. + * @param deserializer + * @param conf + * @param tblProps + * @param partProps + * @throws SerDeException + */ + public static void initializeSerDeWithoutErrorCheck(Deserializer deserializer, + Configuration conf, Properties tblProps, + Properties partProps) throws SerDeException { + if (deserializer instanceof AbstractSerDe) { + ((AbstractSerDe) deserializer).initialize(conf, tblProps, partProps); } else { deserializer.initialize(conf, createOverlayedProperties(tblProps, partProps)); } diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java index 29262ba..07c5ecf 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java @@ -40,7 +40,6 @@ import org.apache.avro.io.BinaryEncoder; import org.apache.avro.io.DecoderFactory; import org.apache.avro.io.EncoderFactory; -import org.apache.avro.util.Utf8; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.common.type.HiveChar; @@ -201,7 +200,7 @@ private Object worker(Object datum, Schema fileSchema, Schema recordSchema, Type // Avro requires NULLable types to be defined as unions of some type T // and NULL. This is annoying and we're going to hide it from the user. if(AvroSerdeUtils.isNullableType(recordSchema)) { - return deserializeNullableUnion(datum, fileSchema, recordSchema, columnType); + return deserializeNullableUnion(datum, fileSchema, recordSchema); } switch(columnType.getCategory()) { @@ -297,8 +296,8 @@ private Object deserializePrimitive(Object datum, Schema fileSchema, Schema reco * Extract either a null or the correct type from a Nullable type. This is * horrible in that we rebuild the TypeInfo every time. */ - private Object deserializeNullableUnion(Object datum, Schema fileSchema, Schema recordSchema, - TypeInfo columnType) throws AvroSerdeException { + private Object deserializeNullableUnion(Object datum, Schema fileSchema, Schema recordSchema) + throws AvroSerdeException { int tag = GenericData.get().resolveUnion(recordSchema, datum); // Determine index of value Schema schema = recordSchema.getTypes().get(tag); if (schema.getType().equals(Schema.Type.NULL)) { @@ -307,8 +306,14 @@ private Object deserializeNullableUnion(Object datum, Schema fileSchema, Schema Schema currentFileSchema = null; if (fileSchema != null) { - currentFileSchema = - fileSchema.getType() == Type.UNION ? fileSchema.getTypes().get(tag) : fileSchema; + if (fileSchema.getType() == Type.UNION) { + // The fileSchema may have the null value in a different position, so + // we need to get the correct tag + tag = GenericData.get().resolveUnion(fileSchema, datum); + currentFileSchema = fileSchema.getTypes().get(tag); + } else { + currentFileSchema = fileSchema; + } } return worker(datum, currentFileSchema, schema, SchemaToTypeInfo.generateTypeInfo(schema)); @@ -370,10 +375,10 @@ private Object deserializeMap(Object datum, Schema fileSchema, Schema mapSchema, // Avro only allows maps with Strings for keys, so we only have to worry // about deserializing the values Map map = new HashMap(); - Map mapDatum = (Map)datum; + Map mapDatum = (Map)datum; Schema valueSchema = mapSchema.getValueType(); TypeInfo valueTypeInfo = columnType.getMapValueTypeInfo(); - for (Utf8 key : mapDatum.keySet()) { + for (CharSequence key : mapDatum.keySet()) { Object value = mapDatum.get(key); map.put(key.toString(), worker(value, fileSchema == null ? null : fileSchema.getValueType(), valueSchema, valueTypeInfo)); diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java index b93121d..9695346 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java @@ -26,8 +26,10 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.AbstractSerDe; import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.SerDeSpec; import org.apache.hadoop.hive.serde2.SerDeStats; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; @@ -37,9 +39,18 @@ /** * Read or write Avro data from Hive. */ +@SerDeSpec(schemaProps = { + serdeConstants.LIST_COLUMNS, serdeConstants.LIST_COLUMN_TYPES, + AvroSerDe.LIST_COLUMN_COMMENTS, AvroSerDe.TABLE_NAME, AvroSerDe.TABLE_COMMENT, + AvroSerdeUtils.SCHEMA_LITERAL, AvroSerdeUtils.SCHEMA_URL, + AvroSerdeUtils.SCHEMA_NAMESPACE, AvroSerdeUtils.SCHEMA_NAME, AvroSerdeUtils.SCHEMA_DOC}) public class AvroSerDe extends AbstractSerDe { private static final Log LOG = LogFactory.getLog(AvroSerDe.class); + public static final String TABLE_NAME = "name"; + public static final String TABLE_COMMENT = "comment"; + public static final String LIST_COLUMN_COMMENTS = "columns.comments"; + public static final String DECIMAL_TYPE_NAME = "decimal"; public static final String CHAR_TYPE_NAME = "char"; public static final String VARCHAR_TYPE_NAME = "varchar"; @@ -59,8 +70,6 @@ private AvroSerializer avroSerializer = null; private boolean badSchema = false; - private static String TABLE_NAME = "name"; - private static String TABLE_COMMENT = "comment"; @Override public void initialize(Configuration configuration, Properties tableProperties, @@ -81,15 +90,15 @@ public void initialize(Configuration configuration, Properties properties) throw columnNames = null; columnTypes = null; - final String columnNameProperty = properties.getProperty("columns"); - final String columnTypeProperty = properties.getProperty("columns.types"); - final String columnCommentProperty = properties.getProperty("columns.comments"); + final String columnNameProperty = properties.getProperty(serdeConstants.LIST_COLUMNS); + final String columnTypeProperty = properties.getProperty(serdeConstants.LIST_COLUMN_TYPES); + final String columnCommentProperty = properties.getProperty(LIST_COLUMN_COMMENTS); if (properties.getProperty(AvroSerdeUtils.SCHEMA_LITERAL) != null || properties.getProperty(AvroSerdeUtils.SCHEMA_URL) != null || columnNameProperty == null || columnNameProperty.isEmpty() || columnTypeProperty == null || columnTypeProperty.isEmpty()) { - schema = AvroSerdeUtils.determineSchemaOrReturnErrorSchema(properties); + schema = determineSchemaOrReturnErrorSchema(properties); } else { // Get column names and sort order columnNames = Arrays.asList(columnNameProperty.split(",")); @@ -135,6 +144,32 @@ public void initialize(Configuration configuration, Properties properties) throw this.oi = aoig.getObjectInspector(); } + /** + * Attempt to determine the schema via the usual means, but do not throw + * an exception if we fail. Instead, signal failure via a special + * schema. This is used because Hive calls init on the serde during + * any call, including calls to update the serde properties, meaning + * if the serde is in a bad state, there is no way to update that state. + */ + public Schema determineSchemaOrReturnErrorSchema(Properties props) { + try { + configErrors = ""; + return AvroSerdeUtils.determineSchemaOrThrowException(props); + } catch(AvroSerdeException he) { + LOG.warn("Encountered AvroSerdeException determining schema. Returning " + + "signal schema to indicate problem", he); + configErrors = new String("Encountered AvroSerdeException determining schema. Returning " + + "signal schema to indicate problem: " + he.getMessage()); + return schema = SchemaResolutionProblem.SIGNAL_BAD_SCHEMA; + } catch (Exception e) { + LOG.warn("Encountered exception determining schema. Returning signal " + + "schema to indicate problem", e); + configErrors = new String("Encountered exception determining schema. Returning signal " + + "schema to indicate problem: " + e.getMessage()); + return SchemaResolutionProblem.SIGNAL_BAD_SCHEMA; + } + } + @Override public Class getSerializedClass() { return AvroGenericRecordWritable.class; diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java index 5da12cb..c0f054f 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java @@ -91,26 +91,6 @@ public static Schema determineSchemaOrThrowException(Properties properties) } } - /** - * Attempt to determine the schema via the usual means, but do not throw - * an exception if we fail. Instead, signal failure via a special - * schema. This is used because Hive calls init on the serde during - * any call, including calls to update the serde properties, meaning - * if the serde is in a bad state, there is no way to update that state. - */ - public static Schema determineSchemaOrReturnErrorSchema(Properties props) { - try { - return determineSchemaOrThrowException(props); - } catch(AvroSerdeException he) { - LOG.warn("Encountered AvroSerdeException determining schema. Returning " + - "signal schema to indicate problem", he); - return SchemaResolutionProblem.SIGNAL_BAD_SCHEMA; - } catch (Exception e) { - LOG.warn("Encountered exception determining schema. Returning signal " + - "schema to indicate problem", e); - return SchemaResolutionProblem.SIGNAL_BAD_SCHEMA; - } - } // Protected for testing and so we can pass in a conf for testing. protected static Schema getSchemaFromFS(String schemaFSUrl, Configuration conf) throws IOException, URISyntaxException { diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java index 06a8c2f..73ae2d9 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hive.serde2.ByteStream.Output; import org.apache.hadoop.hive.serde2.ByteStream.RandomAccessOutput; import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.SerDeSpec; import org.apache.hadoop.hive.serde2.SerDeStats; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DateWritable; @@ -78,7 +79,6 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo; import org.apache.hadoop.io.BooleanWritable; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.FloatWritable; @@ -110,6 +110,9 @@ * fields in the same top-level field will have the same sort order. * */ +@SerDeSpec(schemaProps = { + serdeConstants.LIST_COLUMNS, serdeConstants.LIST_COLUMN_TYPES, + serdeConstants.SERIALIZATION_SORT_ORDER}) public class BinarySortableSerDe extends AbstractSerDe { public static final Log LOG = LogFactory.getLog(BinarySortableSerDe.class.getName()); diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java index 7d6e0d2..1587be8 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java @@ -27,9 +27,11 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; import org.apache.hadoop.hive.serde2.SerDe; import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.SerDeSpec; import org.apache.hadoop.hive.serde2.SerDeUtils; import org.apache.hadoop.hive.serde2.lazy.LazyFactory; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; @@ -50,6 +52,14 @@ * (2) ColumnarSerDe initialize ColumnarStruct's field directly. But under the * field level, it works like LazySimpleSerDe
*/ +@SerDeSpec(schemaProps = { + serdeConstants.LIST_COLUMNS, serdeConstants.LIST_COLUMN_TYPES, + serdeConstants.FIELD_DELIM, serdeConstants.COLLECTION_DELIM, serdeConstants.MAPKEY_DELIM, + serdeConstants.SERIALIZATION_FORMAT, serdeConstants.SERIALIZATION_NULL_FORMAT, + serdeConstants.SERIALIZATION_LAST_COLUMN_TAKES_REST, + serdeConstants.ESCAPE_CHAR, + serdeConstants.SERIALIZATION_ENCODING, + LazySimpleSerDe.SERIALIZATION_EXTEND_NESTING_LEVELS}) public class ColumnarSerDe extends ColumnarSerDeBase { @Override diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarSerDe.java index 86d23f1..87952a1 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarSerDe.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarSerDe.java @@ -22,8 +22,10 @@ import java.util.Properties; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.SerDeSpec; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.SerDeParameters; import org.apache.hadoop.hive.serde2.lazy.LazyUtils; import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryFactory; @@ -44,6 +46,7 @@ * format and which is deserialized in a lazy, i.e. on-demand fashion. * */ +@SerDeSpec(schemaProps = {serdeConstants.LIST_COLUMNS, serdeConstants.LIST_COLUMN_TYPES}) public class LazyBinaryColumnarSerDe extends ColumnarSerDeBase { private List columnNames; diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDe.java index 5c90fd2..21cbd90 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDe.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDe.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hive.serde2.AbstractSerDe; import org.apache.hadoop.hive.serde2.ByteStream; import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.SerDeSpec; import org.apache.hadoop.hive.serde2.SerDeStats; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; @@ -50,6 +51,9 @@ * DynamicSerDe. * */ +@SerDeSpec(schemaProps = { + serdeConstants.SERIALIZATION_DDL, serdeConstants.SERIALIZATION_FORMAT, + DynamicSerDe.META_TABLE_NAME}) public class DynamicSerDe extends AbstractSerDe { public static final Log LOG = LogFactory.getLog(DynamicSerDe.class.getName()); diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/io/DateWritable.java b/serde/src/java/org/apache/hadoop/hive/serde2/io/DateWritable.java index 5289983..2b0b550 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/io/DateWritable.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/io/DateWritable.java @@ -128,7 +128,10 @@ public static Date timeToDate(long l) { public static long daysToMillis(int d) { // Convert from day offset to ms in UTC, then apply local timezone offset. long millisUtc = d * MILLIS_PER_DAY; - return millisUtc - LOCAL_TIMEZONE.get().getOffset(millisUtc); + long tmp = millisUtc - LOCAL_TIMEZONE.get().getOffset(millisUtc); + // Between millisUtc and tmp, the time zone offset may have changed due to DST. + // Look up the offset again. + return millisUtc - LOCAL_TIMEZONE.get().getOffset(tmp); } public static int dateToDays(Date d) { diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/io/TimestampWritable.java b/serde/src/java/org/apache/hadoop/hive/serde2/io/TimestampWritable.java index 0ab27ff..a738b02 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/io/TimestampWritable.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/io/TimestampWritable.java @@ -26,10 +26,7 @@ import java.text.DateFormat; import java.text.SimpleDateFormat; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.common.type.HiveDecimal; -import org.apache.hadoop.hive.serde2.ByteStream.Output; import org.apache.hadoop.hive.serde2.ByteStream.RandomAccessOutput; import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils; import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils.VInt; @@ -55,7 +52,6 @@ * */ public class TimestampWritable implements WritableComparable { - static final private Log LOG = LogFactory.getLog(TimestampWritable.class); static final public byte[] nullBytes = {0x0, 0x0, 0x0, 0x0}; @@ -125,6 +121,12 @@ public void set(byte[] bytes, int offset) { clearTimestamp(); } + public void setTime(long time) { + timestamp.setTime(time); + bytesEmpty = true; + timestampEmpty = false; + } + public void set(Timestamp t) { if (t == null) { timestamp.setTime(0); @@ -301,7 +303,7 @@ public double getDouble() { seconds = getSeconds(); nanos = getNanos(); } - return seconds + ((double) nanos) / 1000000000; + return seconds + nanos / 1000000000; } @@ -453,7 +455,7 @@ public static int getNanos(byte[] bytes, int offset) { /** * Writes a Timestamp's serialized value to byte array b at the given offset - * @param timestamp to convert to bytes + * @param t to convert to bytes * @param b destination byte array * @param offset destination offset in the byte array */ @@ -538,7 +540,7 @@ public static Timestamp doubleToTimestamp(double f) { // We must ensure the exactness of the double's fractional portion. // 0.6 as the fraction part will be converted to 0.59999... and - // significantly reduce the savings from binary serializtion + // significantly reduce the savings from binary serialization BigDecimal bd = new BigDecimal(String.valueOf(f)); bd = bd.subtract(new BigDecimal(seconds)).multiply(new BigDecimal(1000000000)); int nanos = bd.intValue(); diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java index 686fc76..e6932d9 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java @@ -23,6 +23,8 @@ import java.util.Map; import java.util.Set; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyMapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.io.Text; @@ -36,6 +38,7 @@ */ public class LazyMap extends LazyNonPrimitive { + public static final Log LOG = LogFactory.getLog(LazyMap.class); /** * Whether the data is already parsed or not. */ @@ -170,15 +173,19 @@ private void parse() { valueLength[mapSize] = elementByteEnd - (keyEnd[mapSize] + 1); LazyPrimitive lazyKey = uncheckedGetKey(mapSize); if (lazyKey == null) { - continue; - } - Object key = lazyKey.getObject(); - if(!keySet.contains(key)) { - mapSize++; - keySet.add(key); - } else { + LOG.warn("skipped empty entry or entry with empty key in the representation of column with MAP type."); + //reset keyInited[mapSize] flag, since it may be set to true in the case of previous empty entry keyInited[mapSize] = false; + } else { + Object key = lazyKey.getObject(); + if(!keySet.contains(key)) { + mapSize++; + keySet.add(key); + } else { + keyInited[mapSize] = false; + } } + // reset keyValueSeparatorPosition keyValueSeparatorPosition = -1; elementByteBegin = elementByteEnd + 1; diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java index 2c998ef..95e30db 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hive.serde2.lazy; import java.io.IOException; -import java.nio.charset.Charset; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -31,10 +30,10 @@ import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.AbstractEncodingAwareSerDe; -import org.apache.hadoop.hive.serde2.AbstractSerDe; import org.apache.hadoop.hive.serde2.ByteStream; import org.apache.hadoop.hive.serde2.SerDe; import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.SerDeSpec; import org.apache.hadoop.hive.serde2.SerDeStats; import org.apache.hadoop.hive.serde2.SerDeUtils; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; @@ -64,6 +63,14 @@ * Also LazySimpleSerDe outputs typed columns instead of treating all columns as * String like MetadataTypedColumnsetSerDe. */ +@SerDeSpec(schemaProps = { + serdeConstants.LIST_COLUMNS, serdeConstants.LIST_COLUMN_TYPES, + serdeConstants.FIELD_DELIM, serdeConstants.COLLECTION_DELIM, serdeConstants.MAPKEY_DELIM, + serdeConstants.SERIALIZATION_FORMAT, serdeConstants.SERIALIZATION_NULL_FORMAT, + serdeConstants.SERIALIZATION_LAST_COLUMN_TAKES_REST, + serdeConstants.ESCAPE_CHAR, + serdeConstants.SERIALIZATION_ENCODING, + LazySimpleSerDe.SERIALIZATION_EXTEND_NESTING_LEVELS}) public class LazySimpleSerDe extends AbstractEncodingAwareSerDe { public static final Log LOG = LogFactory.getLog(LazySimpleSerDe.class diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java index 6cf9be6..62cba01 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java @@ -27,14 +27,12 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.common.type.Decimal128; -import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.AbstractSerDe; import org.apache.hadoop.hive.serde2.ByteStream; -import org.apache.hadoop.hive.serde2.ByteStream.Output; import org.apache.hadoop.hive.serde2.ByteStream.RandomAccessOutput; import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.SerDeSpec; import org.apache.hadoop.hive.serde2.SerDeStats; import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; @@ -76,6 +74,7 @@ * deserialized until required. Binary means a field is serialized in binary * compact format. */ +@SerDeSpec(schemaProps = {serdeConstants.LIST_COLUMNS, serdeConstants.LIST_COLUMN_TYPES}) public class LazyBinarySerDe extends AbstractSerDe { public static final Log LOG = LogFactory.getLog(LazyBinarySerDe.class.getName()); diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/BaseCharTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/BaseCharTypeInfo.java index 132a79a..7e02f7e 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/BaseCharTypeInfo.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/BaseCharTypeInfo.java @@ -27,6 +27,10 @@ public BaseCharTypeInfo() { } + public BaseCharTypeInfo(String typeName) { + super(typeName); + } + public BaseCharTypeInfo(String typeName, int length) { super(typeName); this.length = length; @@ -53,4 +57,9 @@ public static String getQualifiedName(String typeName, int length) { return sb.toString(); } + @Override + public void setTypeName(String typeName) { + // type name should already be set by subclass + return; + } } diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/CharTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/CharTypeInfo.java index 52fdac2..610818e 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/CharTypeInfo.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/CharTypeInfo.java @@ -25,6 +25,7 @@ // no-arg constructor to make kyro happy. public CharTypeInfo() { + super(serdeConstants.CHAR_TYPE_NAME); } public CharTypeInfo(int length) { diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/DecimalTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/DecimalTypeInfo.java index bd28609..cbe4802 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/DecimalTypeInfo.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/DecimalTypeInfo.java @@ -28,6 +28,7 @@ // no-arg constructor to make kyro happy. public DecimalTypeInfo() { + super(serdeConstants.DECIMAL_TYPE_NAME); } public DecimalTypeInfo(int precision, int scale) { @@ -43,6 +44,12 @@ public String getTypeName() { } @Override + public void setTypeName(String typeName) { + // No need to set type name, it should always be decimal + return; + } + + @Override public boolean equals(Object other) { if (other == null || !(other instanceof DecimalTypeInfo)) { return false; @@ -101,4 +108,20 @@ public boolean accept(TypeInfo other) { return this.precision() - this.scale() >= dti.precision() - dti.scale(); } + public int getPrecision() { + return precision; + } + + public void setPrecision(int precision) { + this.precision = precision; + } + + public int getScale() { + return scale; + } + + public void setScale(int scale) { + this.scale = scale; + } + } diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/StructTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/StructTypeInfo.java index 465bf31..f89a412 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/StructTypeInfo.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/StructTypeInfo.java @@ -102,7 +102,7 @@ public Category getCategory() { public TypeInfo getStructFieldTypeInfo(String field) { String fieldLowerCase = field.toLowerCase(); for (int i = 0; i < allStructFieldNames.size(); i++) { - if (fieldLowerCase.equals(allStructFieldNames.get(i))) { + if (fieldLowerCase.equalsIgnoreCase(allStructFieldNames.get(i))) { return allStructFieldTypeInfos.get(i); } } diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/VarcharTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/VarcharTypeInfo.java index 4550275..5ac2b46 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/VarcharTypeInfo.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/VarcharTypeInfo.java @@ -25,6 +25,7 @@ // no-arg constructor to make kyro happy. public VarcharTypeInfo() { + super(serdeConstants.VARCHAR_TYPE_NAME); } public VarcharTypeInfo(int length) { diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java b/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java index 3a33239..15416a7 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java @@ -475,6 +475,67 @@ public void canDeserializeMapWithNullablePrimitiveValues() throws SerDeException assertTrue(theMap2.containsKey("mu")); assertEquals(null, theMap2.get("mu")); } + + @Test + public void canDeserializeMapsWithJavaLangStringKeys() throws IOException, SerDeException { + // Ensures maps can be deserialized when avro.java.string=String. + // See http://stackoverflow.com/a/19868919/312944 for why that might be used. + String schemaString = "{\n" + + " \"namespace\": \"testing\",\n" + + " \"name\": \"oneMap\",\n" + + " \"type\": \"record\",\n" + + " \"fields\": [\n" + + " {\n" + + " \"name\":\"aMap\",\n" + + " \"type\":{\"type\":\"map\",\n" + + " \"avro.java.string\":\"String\",\n" + + " \"values\":\"long\"}\n" + + "\t}\n" + + " ]\n" + + "}"; + Schema s = AvroSerdeUtils.getSchemaFor(schemaString); + GenericData.Record record = new GenericData.Record(s); + + Map m = new Hashtable(); + m.put("one", 1l); + m.put("two", 2l); + m.put("three", 3l); + + record.put("aMap", m); + assertTrue(GENERIC_DATA.validate(s, record)); + System.out.println("record = " + record); + + AvroGenericRecordWritable garw = Utils.serializeAndDeserializeRecord(record); + + AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s); + + AvroDeserializer de = new AvroDeserializer(); + + ArrayList row = (ArrayList)de.deserialize(aoig.getColumnNames(), + aoig.getColumnTypes(), garw, s); + assertEquals(1, row.size()); + Object theMapObject = row.get(0); + assertTrue(theMapObject instanceof Map); + Map theMap = (Map)theMapObject; + + // Verify the raw object that's been created + assertEquals(1l, theMap.get("one")); + assertEquals(2l, theMap.get("two")); + assertEquals(3l, theMap.get("three")); + + // Verify that the provided object inspector can pull out these same values + StandardStructObjectInspector oi = + (StandardStructObjectInspector)aoig.getObjectInspector(); + + List z = oi.getStructFieldsDataAsList(row); + assertEquals(1, z.size()); + StructField fieldRef = oi.getStructFieldRef("amap"); + + Map theMap2 = (Map)oi.getStructFieldData(row, fieldRef); + assertEquals(1l, theMap2.get("one")); + assertEquals(2l, theMap2.get("two")); + assertEquals(3l, theMap2.get("three")); + } private void verifyNullableType(GenericData.Record record, Schema s, String fieldName, String expected) throws SerDeException, IOException { diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerde.java b/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerde.java index 803a987..36dc484 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerde.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerde.java @@ -85,77 +85,59 @@ public void initializeDoesNotReuseSchemasFromConf() throws SerDeException { } @Test - public void noSchemaProvidedReturnsErrorSchema() throws SerDeException { + public void noSchemaProvidedThrowsException() { Properties props = new Properties(); - verifyErrorSchemaReturned(props); + verifyExpectedException(props); } @Test - public void gibberishSchemaProvidedReturnsErrorSchema() throws SerDeException { + public void gibberishSchemaProvidedReturnsErrorSchema() { Properties props = new Properties(); props.put(AvroSerdeUtils.SCHEMA_LITERAL, "blahblahblah"); - verifyErrorSchemaReturned(props); + verifyExpectedException(props); } @Test - public void emptySchemaProvidedReturnsErrorSchema() throws SerDeException { + public void emptySchemaProvidedThrowsException() { Properties props = new Properties(); props.put(AvroSerdeUtils.SCHEMA_LITERAL, ""); - verifyErrorSchemaReturned(props); + verifyExpectedException(props); } @Test - public void badSchemaURLProvidedReturnsErrorSchema() throws SerDeException { + public void badSchemaURLProvidedThrowsException() { Properties props = new Properties(); props.put(AvroSerdeUtils.SCHEMA_URL, "not://a/url"); - verifyErrorSchemaReturned(props); + verifyExpectedException(props); } @Test - public void emptySchemaURLProvidedReturnsErrorSchema() throws SerDeException { + public void emptySchemaURLProvidedThrowsException() { Properties props = new Properties(); props.put(AvroSerdeUtils.SCHEMA_URL, ""); - verifyErrorSchemaReturned(props); + verifyExpectedException(props); } @Test - public void bothPropertiesSetToNoneReturnsErrorSchema() throws SerDeException { + public void bothPropertiesSetToNoneThrowsException() { Properties props = new Properties(); props.put(AvroSerdeUtils.SCHEMA_URL, AvroSerdeUtils.SCHEMA_NONE); props.put(AvroSerdeUtils.SCHEMA_LITERAL, AvroSerdeUtils.SCHEMA_NONE); - verifyErrorSchemaReturned(props); + verifyExpectedException(props); } - private void verifyErrorSchemaReturned(Properties props) throws SerDeException { + private void verifyExpectedException(Properties props) { AvroSerDe asd = new AvroSerDe(); - SerDeUtils.initializeSerDe(asd, new Configuration(), props, null); - assertTrue(asd.getObjectInspector() instanceof StandardStructObjectInspector); - StandardStructObjectInspector oi = (StandardStructObjectInspector)asd.getObjectInspector(); - List allStructFieldRefs = oi.getAllStructFieldRefs(); - assertEquals(SchemaResolutionProblem.SIGNAL_BAD_SCHEMA.getFields().size(), allStructFieldRefs.size()); - StructField firstField = allStructFieldRefs.get(0); - assertTrue(firstField.toString().contains("error_error_error_error_error_error_error")); - - try { - Writable mock = Mockito.mock(Writable.class); - asd.deserialize(mock); - fail("Should have thrown a BadSchemaException"); - } catch (BadSchemaException bse) { - // good - } - try { - Object o = Mockito.mock(Object.class); - ObjectInspector mockOI = Mockito.mock(ObjectInspector.class); - asd.serialize(o, mockOI); - fail("Should have thrown a BadSchemaException"); - } catch (BadSchemaException bse) { + SerDeUtils.initializeSerDe(asd, new Configuration(), props, null); + fail("Expected Exception did not be thrown"); + } catch (SerDeException e) { // good } } diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/io/TestDateWritable.java b/serde/src/test/org/apache/hadoop/hive/serde2/io/TestDateWritable.java index 75de0a6..8b5bfbd 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/io/TestDateWritable.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/io/TestDateWritable.java @@ -10,6 +10,12 @@ import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Calendar; +import java.util.TimeZone; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; public class TestDateWritable { @@ -135,4 +141,61 @@ public static void setupDateStrings() { private static String getRandomDateString() { return dateStrings[(int) (Math.random() * 365)]; } + + public static class DateTestCallable implements Callable { + public DateTestCallable() { + } + + @Override + public String call() throws Exception { + // Iterate through each day of the year, make sure Date/DateWritable match + Date originalDate = Date.valueOf("2014-01-01"); + Calendar cal = Calendar.getInstance(); + cal.setTimeInMillis(originalDate.getTime()); + for (int idx = 0; idx < 365; ++idx) { + originalDate = new Date(cal.getTimeInMillis()); + // Make sure originalDate is at midnight in the local time zone, + // since DateWritable will generate dates at that time. + originalDate = Date.valueOf(originalDate.toString()); + DateWritable dateWritable = new DateWritable(originalDate); + if (!originalDate.equals(dateWritable.get())) { + return originalDate.toString(); + } + cal.add(Calendar.DAY_OF_YEAR, 1); + } + // Success! + return null; + } + } + + @Test + public void testDaylightSavingsTime() throws InterruptedException, ExecutionException { + String[] timeZones = { + "GMT", + "UTC", + "America/Godthab", + "America/Los_Angeles", + "Asia/Jerusalem", + "Australia/Melbourne", + "Europe/London", + // time zones with half hour boundaries + "America/St_Johns", + "Asia/Tehran", + }; + + for (String timeZone: timeZones) { + TimeZone previousDefault = TimeZone.getDefault(); + TimeZone.setDefault(TimeZone.getTimeZone(timeZone)); + assertEquals("Default timezone should now be " + timeZone, + timeZone, TimeZone.getDefault().getID()); + ExecutorService threadPool = Executors.newFixedThreadPool(1); + try { + Future future = threadPool.submit(new DateTestCallable()); + String result = future.get(); + assertNull("Failed at timezone " + timeZone + ", date " + result, result); + } finally { + threadPool.shutdown(); TimeZone.setDefault(previousDefault); + } + } + } } diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java index ac0b583..48f3b02 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java @@ -194,6 +194,113 @@ public void testLazyMap() throws Throwable { } } + /* + * test LazyMap with bad entries, e.g., empty key or empty entries + * where '[' and ']' don't exist, only for notation purpose, + * STX with value of 2 as entry separator, ETX with 3 as key/value separator + * */ + public void testLazyMapWithBadEntries() throws Throwable { + try { + { + // Map of String to String + Text nullSequence = new Text(""); + ObjectInspector oi = LazyFactory.createLazyObjectInspector( + TypeInfoUtils.getTypeInfosFromTypeString("map").get( + 0), new byte[] {'\2', '\3'}, 0, nullSequence, + false, (byte) 0); + LazyMap b = (LazyMap) LazyFactory.createLazyObject(oi); + + //read friendly string: ak[EXT]av[STX]bk[ETX]bv[STX]ck[ETX]cv[STX]dk[ETX]dv + byte[] data = new byte[] { + 'a', 'k', '\3', 'a', 'v', + '\02', 'b', 'k', '\3', 'b', 'v', + '\02', 'c', 'k', '\3', 'c', 'v', + '\02', 'd', 'k', '\3', 'd', 'v'}; + TestLazyPrimitive.initLazyObject(b, data, 0, data.length); + + assertEquals(new Text("av"), ((LazyString) b + .getMapValueElement(new Text("ak"))).getWritableObject()); + assertNull(b.getMapValueElement(new Text("-1"))); + assertEquals(new Text("bv"), ((LazyString) b + .getMapValueElement(new Text("bk"))).getWritableObject()); + assertEquals(new Text("cv"), ((LazyString) b + .getMapValueElement(new Text("ck"))).getWritableObject()); + assertNull(b.getMapValueElement(new Text("-"))); + assertEquals(new Text("dv"), ((LazyString) b + .getMapValueElement(new Text("dk"))).getWritableObject()); + assertEquals(4, b.getMapSize()); + } + + { + // Map of String to String, LazyMap allows empty-string style key, e.g., {"" : null} + // or {"", ""}, but not null style key, e.g., {null:""} + Text nullSequence = new Text(""); + ObjectInspector oi = LazyFactory.createLazyObjectInspector( + TypeInfoUtils.getTypeInfosFromTypeString("map").get( + 0), new byte[] {'\2', '\3'}, 0, nullSequence, + false, (byte) 0); + LazyMap b = (LazyMap) LazyFactory.createLazyObject(oi); + + //read friendly string: [STX]ak[EXT]av[STX]bk[ETX]bv[STX]ck[ETX]cv[STX]dk[ETX]dv + byte[] data = new byte[] { + '\02', 'a', 'k', '\3', 'a', 'v', + '\02', 'b', 'k', '\3', 'b', 'v', + '\02', 'c', 'k', '\3', 'c', 'v', + '\02', 'd', 'k', '\3', 'd', 'v'}; + TestLazyPrimitive.initLazyObject(b, data, 0, data.length); + + assertNull(b.getMapValueElement(new Text(""))); //{"" : null} + assertEquals(new Text("av"), ((LazyString) b + .getMapValueElement(new Text("ak"))).getWritableObject()); + assertNull(b.getMapValueElement(new Text("-1"))); + assertEquals(new Text("bv"), ((LazyString) b + .getMapValueElement(new Text("bk"))).getWritableObject()); + assertEquals(new Text("cv"), ((LazyString) b + .getMapValueElement(new Text("ck"))).getWritableObject()); + assertNull(b.getMapValueElement(new Text("-"))); + assertEquals(new Text("dv"), ((LazyString) b + .getMapValueElement(new Text("dk"))).getWritableObject()); + assertEquals(4, b.getMapSize()); + } + + { + // Map of String to String, LazyMap allows empty-string style key, e.g., {"" : null} + // or {"", ""}, but not null style key, e.g., {null:""} + Text nullSequence = new Text(""); + ObjectInspector oi = LazyFactory.createLazyObjectInspector( + TypeInfoUtils.getTypeInfosFromTypeString("map").get( + 0), new byte[] {'\2', '\3'}, 0, nullSequence, + false, (byte) 0); + LazyMap b = (LazyMap) LazyFactory.createLazyObject(oi); + + //read friendly string: [ETX][STX]ak[EXT]av[STX]bk[ETX]bv[STX]ck[ETX]cv[STX]dk[ETX]dv + byte[] data = new byte[] { + '\03', + '\02', 'a', 'k', '\3', 'a', 'v', + '\02', 'b', 'k', '\3', 'b', 'v', + '\02', 'c', 'k', '\3', 'c', 'v', + '\02', 'd', 'k', '\3', 'd', 'v'}; + TestLazyPrimitive.initLazyObject(b, data, 0, data.length); + + assertNull(b.getMapValueElement(new Text(""))); //{"" : null} + assertEquals(new Text("av"), ((LazyString) b + .getMapValueElement(new Text("ak"))).getWritableObject()); + assertNull(b.getMapValueElement(new Text("-1"))); + assertEquals(new Text("bv"), ((LazyString) b + .getMapValueElement(new Text("bk"))).getWritableObject()); + assertEquals(new Text("cv"), ((LazyString) b + .getMapValueElement(new Text("ck"))).getWritableObject()); + assertNull(b.getMapValueElement(new Text("-"))); + assertEquals(new Text("dv"), ((LazyString) b + .getMapValueElement(new Text("dk"))).getWritableObject()); + assertEquals(4, b.getMapSize()); + } + } catch(Throwable e) { + e.printStackTrace(); + throw e; + } + } + /** * Test the LazyMap class. */ diff --git a/service/pom.xml b/service/pom.xml index 8a28ffd..9d47a69 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -19,7 +19,7 @@ org.apache.hive hive - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../pom.xml diff --git a/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java b/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java index 83dd2e6..0713449 100644 --- a/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java +++ b/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java @@ -29,10 +29,11 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.shims.HadoopShims.KerberosNameShim; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge; +import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge.Server.ServerMode; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.authentication.util.KerberosName; import org.apache.hive.service.cli.HiveSQLException; import org.apache.hive.service.cli.thrift.ThriftCLIService; import org.apache.thrift.TProcessorFactory; @@ -98,7 +99,7 @@ public HiveAuthFactory(HiveConf conf) throws TTransportException { conf.getVar(ConfVars.HIVE_SERVER2_KERBEROS_PRINCIPAL)); // start delegation token manager try { - saslServer.startDelegationTokenSecretManager(conf, null); + saslServer.startDelegationTokenSecretManager(conf, null, ServerMode.HIVESERVER2); } catch (IOException e) { throw new TTransportException("Failed to start token manager", e); } @@ -234,77 +235,86 @@ public static TServerSocket getServerSSLSocket(String hiveHost, int portNum, Str // retrieve delegation token for the given user public String getDelegationToken(String owner, String renewer) throws HiveSQLException { if (saslServer == null) { - throw new HiveSQLException("Delegation token only supported over kerberos authentication"); + throw new HiveSQLException( + "Delegation token only supported over kerberos authentication", "08S01"); } try { String tokenStr = saslServer.getDelegationTokenWithService(owner, renewer, HS2_CLIENT_TOKEN); if (tokenStr == null || tokenStr.isEmpty()) { - throw new HiveSQLException("Received empty retrieving delegation token for user " + owner); + throw new HiveSQLException( + "Received empty retrieving delegation token for user " + owner, "08S01"); } return tokenStr; } catch (IOException e) { - throw new HiveSQLException("Error retrieving delegation token for user " + owner, e); + throw new HiveSQLException( + "Error retrieving delegation token for user " + owner, "08S01", e); } catch (InterruptedException e) { - throw new HiveSQLException("delegation token retrieval interrupted", e); + throw new HiveSQLException("delegation token retrieval interrupted", "08S01", e); } } // cancel given delegation token public void cancelDelegationToken(String delegationToken) throws HiveSQLException { if (saslServer == null) { - throw new HiveSQLException("Delegation token only supported over kerberos authentication"); + throw new HiveSQLException( + "Delegation token only supported over kerberos authentication", "08S01"); } try { saslServer.cancelDelegationToken(delegationToken); } catch (IOException e) { - throw new HiveSQLException("Error canceling delegation token " + delegationToken, e); + throw new HiveSQLException( + "Error canceling delegation token " + delegationToken, "08S01", e); } } public void renewDelegationToken(String delegationToken) throws HiveSQLException { if (saslServer == null) { - throw new HiveSQLException("Delegation token only supported over kerberos authentication"); + throw new HiveSQLException( + "Delegation token only supported over kerberos authentication", "08S01"); } try { saslServer.renewDelegationToken(delegationToken); } catch (IOException e) { - throw new HiveSQLException("Error renewing delegation token " + delegationToken, e); + throw new HiveSQLException( + "Error renewing delegation token " + delegationToken, "08S01", e); } } public String getUserFromToken(String delegationToken) throws HiveSQLException { if (saslServer == null) { - throw new HiveSQLException("Delegation token only supported over kerberos authentication"); + throw new HiveSQLException( + "Delegation token only supported over kerberos authentication", "08S01"); } try { return saslServer.getUserFromToken(delegationToken); } catch (IOException e) { - throw new HiveSQLException("Error extracting user from delegation token " + delegationToken, - e); + throw new HiveSQLException( + "Error extracting user from delegation token " + delegationToken, "08S01", e); } } public static void verifyProxyAccess(String realUser, String proxyUser, String ipAddress, HiveConf hiveConf) throws HiveSQLException { - try { UserGroupInformation sessionUgi; if (ShimLoader.getHadoopShims().isSecurityEnabled()) { - KerberosName kerbName = new KerberosName(realUser); - String shortPrincipalName = kerbName.getServiceName(); + KerberosNameShim kerbName = ShimLoader.getHadoopShims().getKerberosNameShim(realUser); + String shortPrincipalName = kerbName.getServiceName(); sessionUgi = ShimLoader.getHadoopShims().createProxyUser(shortPrincipalName); } else { sessionUgi = ShimLoader.getHadoopShims().createRemoteUser(realUser, null); } + Thread.dumpStack(); + System.err.println("realUser = " + realUser + ", proxyUser = " + proxyUser + ", sessionUgi = " + sessionUgi); if (!proxyUser.equalsIgnoreCase(realUser)) { ShimLoader.getHadoopShims(). authorizeProxyAccess(proxyUser, sessionUgi, ipAddress, hiveConf); } } catch (IOException e) { throw new HiveSQLException( - "Failed to validate proxy privilege of " + realUser + " for " + proxyUser, e); + "Failed to validate proxy privilege of " + realUser + " for " + proxyUser, "08S01", e); } } - + } diff --git a/service/src/java/org/apache/hive/service/cli/CLIService.java b/service/src/java/org/apache/hive/service/cli/CLIService.java index b46c5b4..f5751f1 100644 --- a/service/src/java/org/apache/hive/service/cli/CLIService.java +++ b/service/src/java/org/apache/hive/service/cli/CLIService.java @@ -44,6 +44,7 @@ import org.apache.hive.service.cli.operation.Operation; import org.apache.hive.service.cli.session.SessionManager; import org.apache.hive.service.cli.thrift.TProtocolVersion; +import org.apache.hive.service.server.HiveServer2; /** * CLIService. @@ -64,15 +65,18 @@ private SessionManager sessionManager; private UserGroupInformation serviceUGI; private UserGroupInformation httpUGI; + // The HiveServer2 instance running this service + private final HiveServer2 hiveServer2; - public CLIService() { + public CLIService(HiveServer2 hiveServer2) { super(CLIService.class.getSimpleName()); + this.hiveServer2 = hiveServer2; } @Override public synchronized void init(HiveConf hiveConf) { this.hiveConf = hiveConf; - sessionManager = new SessionManager(); + sessionManager = new SessionManager(hiveServer2); addService(sessionManager); // If the hadoop cluster is secure, do a kerberos login for the service from the keytab if (ShimLoader.getHadoopShims().isSecurityEnabled()) { @@ -201,7 +205,8 @@ public SessionHandle openSessionWithImpersonation(String username, String passwo * @see org.apache.hive.service.cli.ICLIService#closeSession(org.apache.hive.service.cli.SessionHandle) */ @Override - public void closeSession(SessionHandle sessionHandle) throws HiveSQLException { + public void closeSession(SessionHandle sessionHandle) + throws HiveSQLException { sessionManager.closeSession(sessionHandle); LOG.debug(sessionHandle + ": closeSession()"); } diff --git a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java index f021870..cabf32a 100644 --- a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java +++ b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java @@ -35,14 +35,16 @@ import org.apache.hadoop.hive.common.cli.IHiveFileProcessor; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.ql.exec.FetchFormatter; import org.apache.hadoop.hive.ql.exec.ListSinkOperator; import org.apache.hadoop.hive.ql.history.HiveHistory; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.processors.SetProcessor; import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hive.common.util.HiveVersionInfo; import org.apache.hive.service.auth.HiveAuthFactory; import org.apache.hive.service.cli.*; @@ -80,7 +82,6 @@ private SessionManager sessionManager; private OperationManager operationManager; - private IMetaStoreClient metastoreClient = null; private final Set opHandleSet = new HashSet(); private boolean isOperationLogEnabled; private File sessionLogDir; @@ -95,6 +96,17 @@ public HiveSessionImpl(TProtocolVersion protocol, String username, String passwo this.hiveConf = new HiveConf(serverhiveConf); this.ipAddress = ipAddress; + try { + // In non-impersonation mode, map scheduler queue to current user + // if fair scheduler is configured. + if (! hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS) && + hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_MAP_FAIR_SCHEDULER_QUEUE)) { + ShimLoader.getHadoopShims().refreshDefaultQueue(hiveConf, username); + } + } catch (IOException e) { + LOG.warn("Error setting scheduler queue: " + e, e); + } + // Set an explicit session name to control the download directory name hiveConf.set(ConfVars.HIVESESSIONID.varname, sessionHandle.getHandleIdentifier().toString()); @@ -315,14 +327,13 @@ public HiveConf getHiveConf() { @Override public IMetaStoreClient getMetaStoreClient() throws HiveSQLException { - if (metastoreClient == null) { - try { - metastoreClient = new HiveMetaStoreClient(getHiveConf()); - } catch (MetaException e) { - throw new HiveSQLException(e); - } + try { + return Hive.get(getHiveConf()).getMSC(); + } catch (HiveException e) { + throw new HiveSQLException("Failed to get metastore connection", e); + } catch (MetaException e) { + throw new HiveSQLException("Failed to get metastore connection", e); } - return metastoreClient; } @Override @@ -538,14 +549,6 @@ public OperationHandle getFunctions(String catalogName, String schemaName, Strin public void close() throws HiveSQLException { try { acquire(true); - /** - * For metadata operations like getTables(), getColumns() etc, - * the session allocates a private metastore handler which should be - * closed at the end of the session - */ - if (metastoreClient != null) { - metastoreClient.close(); - } // Iterate through the opHandles and close their operations for (OperationHandle opHandle : opHandleSet) { operationManager.closeOperation(opHandle); diff --git a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java index bad533b..5dfcd06 100644 --- a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java +++ b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java @@ -152,6 +152,7 @@ public void setProxySession(HiveSession proxySession) { @Override public String getDelegationToken(HiveAuthFactory authFactory, String owner, String renewer) throws HiveSQLException { + HiveAuthFactory.verifyProxyAccess(getUsername(), owner, getIpAddress(), getHiveConf()); return authFactory.getDelegationToken(owner, renewer); } diff --git a/service/src/java/org/apache/hive/service/cli/session/SessionManager.java b/service/src/java/org/apache/hive/service/cli/session/SessionManager.java index ecc9b96..137359b 100644 --- a/service/src/java/org/apache/hive/service/cli/session/SessionManager.java +++ b/service/src/java/org/apache/hive/service/cli/session/SessionManager.java @@ -43,6 +43,7 @@ import org.apache.hive.service.cli.SessionHandle; import org.apache.hive.service.cli.operation.OperationManager; import org.apache.hive.service.cli.thrift.TProtocolVersion; +import org.apache.hive.service.server.HiveServer2; import org.apache.hive.service.server.ThreadFactoryWithGarbageCleanup; /** @@ -65,9 +66,12 @@ private long sessionTimeout; private volatile boolean shutdown; + // The HiveServer2 instance running this service + private final HiveServer2 hiveServer2; - public SessionManager() { + public SessionManager(HiveServer2 hiveServer2) { super(SessionManager.class.getSimpleName()); + this.hiveServer2 = hiveServer2; } @Override @@ -232,10 +236,10 @@ public SessionHandle openSession(TProtocolVersion protocol, String username, Str /** * Opens a new session and creates a session handle. * The username passed to this method is the effective username. - * If withImpersonation is true (==doAs true) we wrap all the calls in HiveSession + * If withImpersonation is true (==doAs true) we wrap all the calls in HiveSession * within a UGI.doAs, where UGI corresponds to the effective user. - * @see org.apache.hive.service.cli.thrift.ThriftCLIService#getUserName() - * + * @see org.apache.hive.service.cli.thrift.ThriftCLIService#getUserName() + * * @param protocol * @param username * @param password @@ -288,6 +292,24 @@ public void closeSession(SessionHandle sessionHandle) throws HiveSQLException { throw new HiveSQLException("Session does not exist!"); } session.close(); + // Shutdown HiveServer2 if it has been deregistered from ZooKeeper and has no active sessions + if (!(hiveServer2 == null) && (hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_SUPPORT_DYNAMIC_SERVICE_DISCOVERY)) + && (!hiveServer2.isRegisteredWithZooKeeper())) { + // Asynchronously shutdown this instance of HiveServer2, + // if there are no active client sessions + if (getOpenSessionCount() == 0) { + LOG.info("This instance of HiveServer2 has been removed from the list of server " + + "instances available for dynamic service discovery. " + + "The last client session has ended - will shutdown now."); + Thread shutdownThread = new Thread() { + @Override + public void run() { + hiveServer2.stop(); + } + }; + shutdownThread.start(); + } + } } public HiveSession getSession(SessionHandle sessionHandle) throws HiveSQLException { @@ -376,6 +398,5 @@ private void executeSessionHooks(HiveSession session) throws Exception { public int getOpenSessionCount() { return handleToSession.size(); } - } diff --git a/service/src/java/org/apache/hive/service/cli/thrift/EmbeddedThriftBinaryCLIService.java b/service/src/java/org/apache/hive/service/cli/thrift/EmbeddedThriftBinaryCLIService.java index 9ee9785..61700c1 100644 --- a/service/src/java/org/apache/hive/service/cli/thrift/EmbeddedThriftBinaryCLIService.java +++ b/service/src/java/org/apache/hive/service/cli/thrift/EmbeddedThriftBinaryCLIService.java @@ -30,7 +30,7 @@ public class EmbeddedThriftBinaryCLIService extends ThriftBinaryCLIService { public EmbeddedThriftBinaryCLIService() { - super(new CLIService()); + super(new CLIService(null)); isEmbedded = true; HiveConf.setLoadHiveServer2Config(true); cliService.init(new HiveConf()); diff --git a/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java b/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java index 4a1e004..a0a6e18 100644 --- a/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java +++ b/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java @@ -66,7 +66,6 @@ protected int minWorkerThreads; protected int maxWorkerThreads; protected long workerKeepAliveTime; - private HiveServer2 hiveServer2; public ThriftCLIService(CLIService cliService, String serviceName) { super(serviceName); @@ -264,9 +263,9 @@ private String getIpAddress() { /** * Returns the effective username. - * 1. If hive.server2.allow.user.substitution = false: the username of the connecting user + * 1. If hive.server2.allow.user.substitution = false: the username of the connecting user * 2. If hive.server2.allow.user.substitution = true: the username of the end user, - * that the connecting user is trying to proxy for. + * that the connecting user is trying to proxy for. * This includes a check whether the connecting user is allowed to proxy for the end user. * @param req * @return @@ -366,24 +365,6 @@ public TCloseSessionResp CloseSession(TCloseSessionReq req) throws TException { } catch (Exception e) { LOG.warn("Error closing session: ", e); resp.setStatus(HiveSQLException.toTStatus(e)); - } finally { - if (!(isEmbedded) && (hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_SUPPORT_DYNAMIC_SERVICE_DISCOVERY)) - && (!hiveServer2.isRegisteredWithZooKeeper())) { - // Asynchronously shutdown this instance of HiveServer2, - // if there are no active client sessions - if (cliService.getSessionManager().getOpenSessionCount() == 0) { - LOG.info("This instance of HiveServer2 has been removed from the list of server " - + "instances available for dynamic service discovery. " - + "The last client session has ended - will shutdown now."); - Thread shutdownThread = new Thread() { - @Override - public void run() { - hiveServer2.stop(); - } - }; - shutdownThread.start(); - } - } } return resp; } @@ -666,10 +647,4 @@ private boolean isKerberosAuthMode() { return cliService.getHiveConf().getVar(ConfVars.HIVE_SERVER2_AUTHENTICATION) .equalsIgnoreCase(HiveAuthFactory.AuthTypes.KERBEROS.toString()); } - - public void setHiveServer2(HiveServer2 hiveServer2) { - this.hiveServer2 = hiveServer2; - } - } - diff --git a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java index cfa7284..f7b1648 100644 --- a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java +++ b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java @@ -29,12 +29,10 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Shell; import org.apache.hive.service.auth.HiveAuthFactory; -import org.apache.hive.service.auth.HiveAuthFactory.AuthTypes; import org.apache.hive.service.cli.CLIService; import org.apache.hive.service.cli.thrift.TCLIService.Iface; import org.apache.hive.service.server.ThreadFactoryWithGarbageCleanup; import org.apache.thrift.TProcessor; -import org.apache.thrift.TProcessorFactory; import org.apache.thrift.protocol.TBinaryProtocol; import org.apache.thrift.protocol.TProtocolFactory; import org.apache.thrift.server.TServlet; @@ -60,9 +58,6 @@ public ThriftHttpCLIService(CLIService cliService) { @Override public void run() { try { - // Verify config validity - verifyHttpConfiguration(hiveConf); - // HTTP Server httpServer = new org.eclipse.jetty.server.Server(); @@ -162,32 +157,4 @@ private String getHttpPath(String httpPath) { } return httpPath; } - - /** - * Verify that this configuration is supported by transportMode of HTTP - * @param hiveConf - */ - private static void verifyHttpConfiguration(HiveConf hiveConf) { - String authType = hiveConf.getVar(ConfVars.HIVE_SERVER2_AUTHENTICATION); - - // Error out if KERBEROS auth mode is being used and use SSL is also set to true - if(authType.equalsIgnoreCase(AuthTypes.KERBEROS.toString()) && - hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_USE_SSL)) { - String msg = ConfVars.HIVE_SERVER2_AUTHENTICATION + " setting of " + - authType + " is not supported with " + - ConfVars.HIVE_SERVER2_USE_SSL + " set to true"; - LOG.fatal(msg); - throw new RuntimeException(msg); - } - - // Warn that SASL is not used in http mode - if(authType.equalsIgnoreCase(AuthTypes.NONE.toString())) { - // NONE in case of thrift mode uses SASL - LOG.warn(ConfVars.HIVE_SERVER2_AUTHENTICATION + " setting to " + - authType + ". SASL is not supported with http transport mode," + - " so using equivalent of " - + AuthTypes.NOSASL); - } - } - } diff --git a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java index 312d05e..c61d3a3 100644 --- a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java +++ b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java @@ -31,8 +31,9 @@ import org.apache.commons.codec.binary.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.shims.HadoopShims.KerberosNameShim; +import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.authentication.util.KerberosName; import org.apache.hive.service.auth.AuthenticationProviderFactory; import org.apache.hive.service.auth.AuthenticationProviderFactory.AuthMethods; import org.apache.hive.service.auth.HiveAuthFactory; @@ -237,19 +238,31 @@ public String run() throws HttpAuthenticationException { } } - private String getPrincipalWithoutRealm(String fullPrincipal) { - KerberosName fullKerberosName = new KerberosName(fullPrincipal); + private String getPrincipalWithoutRealm(String fullPrincipal) + throws HttpAuthenticationException { + KerberosNameShim fullKerberosName; + try { + fullKerberosName = ShimLoader.getHadoopShims().getKerberosNameShim(fullPrincipal); + } catch (IOException e) { + throw new HttpAuthenticationException(e); + } String serviceName = fullKerberosName.getServiceName(); - String hostName = fullKerberosName.getHostName(); + String hostName = fullKerberosName.getHostName(); String principalWithoutRealm = serviceName; if (hostName != null) { principalWithoutRealm = serviceName + "/" + hostName; } return principalWithoutRealm; } - - private String getPrincipalWithoutRealmAndHost(String fullPrincipal) { - KerberosName fullKerberosName = new KerberosName(fullPrincipal); + + private String getPrincipalWithoutRealmAndHost(String fullPrincipal) + throws HttpAuthenticationException { + KerberosNameShim fullKerberosName; + try { + fullKerberosName = ShimLoader.getHadoopShims().getKerberosNameShim(fullPrincipal); + } catch (IOException e) { + throw new HttpAuthenticationException(e); + } return fullKerberosName.getServiceName(); } } diff --git a/service/src/java/org/apache/hive/service/server/HiveServer2.java b/service/src/java/org/apache/hive/service/server/HiveServer2.java index 3d2356b..1e6785b 100644 --- a/service/src/java/org/apache/hive/service/server/HiveServer2.java +++ b/service/src/java/org/apache/hive/service/server/HiveServer2.java @@ -18,8 +18,18 @@ package org.apache.hive.service.server; +import java.io.IOException; import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionBuilder; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.common.LogUtils; @@ -29,10 +39,10 @@ import org.apache.hadoop.hive.ql.exec.spark.session.SparkSessionManagerImpl; import org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolManager; import org.apache.hadoop.hive.ql.util.ZooKeeperHiveHelper; +import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hive.common.util.HiveStringUtils; import org.apache.hive.common.util.HiveVersionInfo; import org.apache.hive.service.CompositeService; -import org.apache.hive.service.ServiceException; import org.apache.hive.service.cli.CLIService; import org.apache.hive.service.cli.thrift.ThriftBinaryCLIService; import org.apache.hive.service.cli.thrift.ThriftCLIService; @@ -42,7 +52,9 @@ import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.Watcher; import org.apache.zookeeper.ZooDefs.Ids; +import org.apache.zookeeper.ZooDefs.Perms; import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.data.ACL; /** * HiveServer2. @@ -65,7 +77,7 @@ public HiveServer2() { @Override public synchronized void init(HiveConf hiveConf) { - cliService = new CLIService(); + cliService = new CLIService(this); addService(cliService); if (isHTTPTransportMode(hiveConf)) { thriftCLIService = new ThriftHttpCLIService(cliService); @@ -73,7 +85,6 @@ public synchronized void init(HiveConf hiveConf) { thriftCLIService = new ThriftBinaryCLIService(cliService); } addService(thriftCLIService); - thriftCLIService.setHiveServer2(this); super.init(hiveConf); // Add a shutdown hook for catching SIGTERM & SIGINT @@ -110,14 +121,19 @@ private void addServerInstanceToZooKeeper(HiveConf hiveConf) throws Exception { String rootNamespace = hiveConf.getVar(HiveConf.ConfVars.HIVE_SERVER2_ZOOKEEPER_NAMESPACE); String instanceURI = getServerInstanceURI(hiveConf); byte[] znodeDataUTF8 = instanceURI.getBytes(Charset.forName("UTF-8")); + // Znode ACLs + List nodeAcls = new ArrayList(); + setUpAuthAndAcls(hiveConf, nodeAcls); + // Create a ZooKeeper client zooKeeperClient = new ZooKeeper(zooKeeperEnsemble, zooKeeperSessionTimeout, new ZooKeeperHiveHelper.DummyWatcher()); - - // Create the parent znodes recursively; ignore if the parent already exists + // Create the parent znodes recursively; ignore if the parent already exists. + // If pre-creating the parent on a kerberized cluster, ensure that you give ACLs, + // as explained in {@link #setUpAuthAndAcls(HiveConf, List) setUpAuthAndAcls} try { - ZooKeeperHiveHelper.createPathRecursively(zooKeeperClient, rootNamespace, - Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + ZooKeeperHiveHelper.createPathRecursively(zooKeeperClient, rootNamespace, nodeAcls, + CreateMode.PERSISTENT); LOG.info("Created the root name space: " + rootNamespace + " on ZooKeeper for HiveServer2"); } catch (KeeperException e) { if (e.code() != KeeperException.Code.NODEEXISTS) { @@ -126,14 +142,14 @@ private void addServerInstanceToZooKeeper(HiveConf hiveConf) throws Exception { } } // Create a znode under the rootNamespace parent for this instance of the server - // Znode name: server-host:port-versionInfo-sequence + // Znode name: serverUri=host:port;version=versionInfo;sequence=sequenceNumber try { - String znodePath = + String pathPrefix = ZooKeeperHiveHelper.ZOOKEEPER_PATH_SEPARATOR + rootNamespace - + ZooKeeperHiveHelper.ZOOKEEPER_PATH_SEPARATOR + "server-" + instanceURI + "-" - + HiveVersionInfo.getVersion() + "-"; + + ZooKeeperHiveHelper.ZOOKEEPER_PATH_SEPARATOR + "serverUri=" + instanceURI + ";" + + "version=" + HiveVersionInfo.getVersion() + ";" + "sequence="; znodePath = - zooKeeperClient.create(znodePath, znodeDataUTF8, Ids.OPEN_ACL_UNSAFE, + zooKeeperClient.create(pathPrefix, znodeDataUTF8, nodeAcls, CreateMode.EPHEMERAL_SEQUENTIAL); setRegisteredWithZooKeeper(true); // Set a watch on the znode @@ -149,11 +165,50 @@ private void addServerInstanceToZooKeeper(HiveConf hiveConf) throws Exception { } /** + * Set up ACLs for znodes based on whether the cluster is secure or not. + * On a kerberized cluster, ZooKeeper performs Kerberos-SASL authentication. + * We give Read privilege to the world, but Create/Delete/Write/Admin to the authenticated user. + * On a non-kerberized cluster, we give Create/Read/Delete/Write/Admin privileges to the world. + * + * For a kerberized cluster, we also dynamically set up the client's JAAS conf. + * @param hiveConf + * @param nodeAcls + * @return + * @throws Exception + */ + private void setUpAuthAndAcls(HiveConf hiveConf, List nodeAcls) throws Exception { + if (ShimLoader.getHadoopShims().isSecurityEnabled()) { + String principal = hiveConf.getVar(ConfVars.HIVE_SERVER2_KERBEROS_PRINCIPAL); + if (principal.isEmpty()) { + throw new IOException( + "HiveServer2 Kerberos principal is empty"); + } + String keyTabFile = hiveConf.getVar(ConfVars.HIVE_SERVER2_KERBEROS_KEYTAB); + if (keyTabFile.isEmpty()) { + throw new IOException( + "HiveServer2 Kerberos keytab is empty"); + } + + // Install the JAAS Configuration for the runtime + ShimLoader.getHadoopShims().setZookeeperClientKerberosJaasConfig(principal, keyTabFile); + // Read all to the world + nodeAcls.addAll(Ids.READ_ACL_UNSAFE); + // Create/Delete/Write/Admin to the authenticated user + nodeAcls.add(new ACL(Perms.ALL, Ids.AUTH_IDS)); + } else { + // ACLs for znodes on a non-kerberized cluster + // Create/Read/Delete/Write/Admin to the world + nodeAcls.addAll(Ids.OPEN_ACL_UNSAFE); + } + } + + /** * The watcher class which sets the de-register flag when the znode corresponding to this server * instance is deleted. Additionally, it shuts down the server if there are no more active client * sessions at the time of receiving a 'NodeDeleted' notification from ZooKeeper. */ private class DeRegisterWatcher implements Watcher { + @Override public void process(WatchedEvent event) { if (event.getType().equals(Watcher.Event.EventType.NodeDeleted)) { HiveServer2.this.setRegisteredWithZooKeeper(false); @@ -233,6 +288,7 @@ public synchronized void stop() { private static void startHiveServer2() throws Throwable { long attempts = 0, maxAttempts = 1; while (true) { + LOG.info("Starting HiveServer2"); HiveConf hiveConf = new HiveConf(); maxAttempts = hiveConf.getLongVar(HiveConf.ConfVars.HIVE_SERVER2_MAX_START_ATTEMPTS); HiveServer2 server = null; @@ -280,31 +336,206 @@ private static void startHiveServer2() throws Throwable { } } + /** + * Remove all znodes corresponding to the given version number from ZooKeeper + * + * @param versionNumber + * @throws Exception + */ + static void deleteServerInstancesFromZooKeeper(String versionNumber) throws Exception { + HiveConf hiveConf = new HiveConf(); + int zooKeeperSessionTimeout = + hiveConf.getIntVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_SESSION_TIMEOUT); + String zooKeeperEnsemble = ZooKeeperHiveHelper.getQuorumServers(hiveConf); + String rootNamespace = hiveConf.getVar(HiveConf.ConfVars.HIVE_SERVER2_ZOOKEEPER_NAMESPACE); + ZooKeeper zooKeeperClient = + new ZooKeeper(zooKeeperEnsemble, zooKeeperSessionTimeout, + new ZooKeeperHiveHelper.DummyWatcher()); + // Get all znode paths + List znodePaths = + zooKeeperClient.getChildren(ZooKeeperHiveHelper.ZOOKEEPER_PATH_SEPARATOR + rootNamespace, + false); + // Now for each path that is for the given versionNumber, delete the znode from ZooKeeper + for (String znodePath : znodePaths) { + if (znodePath.contains("version=" + versionNumber + ";")) { + zooKeeperClient.delete(ZooKeeperHiveHelper.ZOOKEEPER_PATH_SEPARATOR + rootNamespace + + ZooKeeperHiveHelper.ZOOKEEPER_PATH_SEPARATOR + znodePath, -1); + } + } + } + public static void main(String[] args) { HiveConf.setLoadHiveServer2Config(true); try { ServerOptionsProcessor oproc = new ServerOptionsProcessor("hiveserver2"); - if (!oproc.process(args)) { - System.err.println("Error starting HiveServer2 with given arguments"); - System.exit(-1); - } + ServerOptionsProcessorResponse oprocResponse = oproc.parse(args); // NOTE: It is critical to do this here so that log4j is reinitialized // before any of the other core hive classes are loaded String initLog4jMessage = LogUtils.initHiveLog4j(); LOG.debug(initLog4jMessage); - HiveStringUtils.startupShutdownMessage(HiveServer2.class, args, LOG); - // log debug message from "oproc" after log4j initialize properly + + // Log debug message from "oproc" after log4j initialize properly LOG.debug(oproc.getDebugMessage().toString()); - startHiveServer2(); + // Call the executor which will execute the appropriate command based on the parsed options + oprocResponse.getServerOptionsExecutor().execute(); } catch (LogInitializationException e) { LOG.error("Error initializing log: " + e.getMessage(), e); System.exit(-1); - } catch (Throwable t) { - LOG.fatal("Error starting HiveServer2", t); - System.exit(-1); + } + } + + /** + * ServerOptionsProcessor. + * Process arguments given to HiveServer2 (-hiveconf property=value) + * Set properties in System properties + * Create an appropriate response object, + * which has executor to execute the appropriate command based on the parsed options. + */ + static class ServerOptionsProcessor { + private final Options options = new Options(); + private org.apache.commons.cli.CommandLine commandLine; + private final String serverName; + private final StringBuilder debugMessage = new StringBuilder(); + + @SuppressWarnings("static-access") + ServerOptionsProcessor(String serverName) { + this.serverName = serverName; + // -hiveconf x=y + options.addOption(OptionBuilder + .withValueSeparator() + .hasArgs(2) + .withArgName("property=value") + .withLongOpt("hiveconf") + .withDescription("Use value for given property") + .create()); + // -deregister + options.addOption(OptionBuilder + .hasArgs(1) + .withArgName("versionNumber") + .withLongOpt("deregister") + .withDescription("Deregister all instances of given version from dynamic service discovery") + .create()); + options.addOption(new Option("H", "help", false, "Print help information")); + } + + ServerOptionsProcessorResponse parse(String[] argv) { + try { + commandLine = new GnuParser().parse(options, argv); + // Process --hiveconf + // Get hiveconf param values and set the System property values + Properties confProps = commandLine.getOptionProperties("hiveconf"); + for (String propKey : confProps.stringPropertyNames()) { + // save logging message for log4j output latter after log4j initialize properly + debugMessage.append("Setting " + propKey + "=" + confProps.getProperty(propKey) + ";\n"); + System.setProperty(propKey, confProps.getProperty(propKey)); + } + + // Process --help + if (commandLine.hasOption('H')) { + return new ServerOptionsProcessorResponse(new HelpOptionExecutor(serverName, options)); + } + + // Process --deregister + if (commandLine.hasOption("deregister")) { + return new ServerOptionsProcessorResponse(new DeregisterOptionExecutor( + commandLine.getOptionValue("deregister"))); + } + } catch (ParseException e) { + // Error out & exit - we were not able to parse the args successfully + System.err.println("Error starting HiveServer2 with given arguments: "); + System.err.println(e.getMessage()); + System.exit(-1); + } + // Default executor, when no option is specified + return new ServerOptionsProcessorResponse(new StartOptionExecutor()); + } + + StringBuilder getDebugMessage() { + return debugMessage; + } + } + + /** + * The response sent back from {@link ServerOptionsProcessor#parse(String[])} + */ + static class ServerOptionsProcessorResponse { + private final ServerOptionsExecutor serverOptionsExecutor; + + ServerOptionsProcessorResponse(ServerOptionsExecutor serverOptionsExecutor) { + this.serverOptionsExecutor = serverOptionsExecutor; + } + + ServerOptionsExecutor getServerOptionsExecutor() { + return serverOptionsExecutor; + } + } + + /** + * The executor interface for running the appropriate HiveServer2 command based on parsed options + */ + static interface ServerOptionsExecutor { + public void execute(); + } + + /** + * HelpOptionExecutor: executes the --help option by printing out the usage + */ + static class HelpOptionExecutor implements ServerOptionsExecutor { + private final Options options; + private final String serverName; + + HelpOptionExecutor(String serverName, Options options) { + this.options = options; + this.serverName = serverName; + } + + @Override + public void execute() { + new HelpFormatter().printHelp(serverName, options); + System.exit(0); + } + } + + /** + * StartOptionExecutor: starts HiveServer2. + * This is the default executor, when no option is specified. + */ + static class StartOptionExecutor implements ServerOptionsExecutor { + @Override + public void execute() { + try { + startHiveServer2(); + } catch (Throwable t) { + LOG.fatal("Error starting HiveServer2", t); + System.exit(-1); + } + } + } + + /** + * DeregisterOptionExecutor: executes the --deregister option by + * deregistering all HiveServer2 instances from ZooKeeper of a specific version. + */ + static class DeregisterOptionExecutor implements ServerOptionsExecutor { + private final String versionNumber; + + DeregisterOptionExecutor(String versionNumber) { + this.versionNumber = versionNumber; + } + + @Override + public void execute() { + try { + deleteServerInstancesFromZooKeeper(versionNumber); + } catch (Exception e) { + LOG.fatal("Error deregistering HiveServer2 instances for version: " + versionNumber + + " from ZooKeeper", e); + System.exit(-1); + } + System.exit(0); } } } diff --git a/service/src/java/org/apache/hive/service/server/ServerOptionsProcessor.java b/service/src/java/org/apache/hive/service/server/ServerOptionsProcessor.java index bbb2a42..e69de29 100644 --- a/service/src/java/org/apache/hive/service/server/ServerOptionsProcessor.java +++ b/service/src/java/org/apache/hive/service/server/ServerOptionsProcessor.java @@ -1,91 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hive.service.server; - -import java.util.Properties; - -import org.apache.commons.cli.GnuParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.OptionBuilder; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -/** - * ServerOptionsProcessor. - * Process arguments given to servers (-hiveconf property=value) - * Set properties in System properties - */ -public class ServerOptionsProcessor { - protected static final Log LOG = LogFactory.getLog(ServerOptionsProcessor.class.getName()); - private final Options options = new Options(); - private org.apache.commons.cli.CommandLine commandLine; - private final String serverName; - private StringBuilder debugMessage = new StringBuilder(); - - - @SuppressWarnings("static-access") - public ServerOptionsProcessor(String serverName) { - this.serverName = serverName; - // -hiveconf x=y - options.addOption(OptionBuilder - .withValueSeparator() - .hasArgs(2) - .withArgName("property=value") - .withLongOpt("hiveconf") - .withDescription("Use value for given property") - .create()); - - options.addOption(new Option("H", "help", false, "Print help information")); - - } - - public boolean process(String[] argv) { - try { - commandLine = new GnuParser().parse(options, argv); - if (commandLine.hasOption('H')) { - printUsage(); - return false; - } - //get hiveconf param values and set the System property values - Properties confProps = commandLine.getOptionProperties("hiveconf"); - for (String propKey : confProps.stringPropertyNames()) { - //save logging message for log4j output latter after log4j initialize properly - debugMessage.append("Setting " + propKey + "=" + confProps.getProperty(propKey) + ";\n"); - System.setProperty(propKey, confProps.getProperty(propKey)); - } - } catch (ParseException e) { - System.err.println(e.getMessage()); - printUsage(); - return false; - } - return true; - } - - public StringBuilder getDebugMessage() { - return debugMessage; - } - - private void printUsage() { - new HelpFormatter().printHelp(serverName, options); - } - -} diff --git a/service/src/test/org/apache/hive/service/auth/TestPlainSaslHelper.java b/service/src/test/org/apache/hive/service/auth/TestPlainSaslHelper.java index fb784aae..03f3964 100644 --- a/service/src/test/org/apache/hive/service/auth/TestPlainSaslHelper.java +++ b/service/src/test/org/apache/hive/service/auth/TestPlainSaslHelper.java @@ -39,7 +39,7 @@ public void testDoAsSetting(){ hconf.getBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS)); - CLIService cliService = new CLIService(); + CLIService cliService = new CLIService(null); cliService.init(hconf); ThriftCLIService tcliService = new ThriftBinaryCLIService(cliService); tcliService.init(hconf); diff --git a/service/src/test/org/apache/hive/service/cli/session/TestSessionGlobalInitFile.java b/service/src/test/org/apache/hive/service/cli/session/TestSessionGlobalInitFile.java index 47d3a56..37b698b 100644 --- a/service/src/test/org/apache/hive/service/cli/session/TestSessionGlobalInitFile.java +++ b/service/src/test/org/apache/hive/service/cli/session/TestSessionGlobalInitFile.java @@ -52,7 +52,7 @@ */ private class FakeEmbeddedThriftBinaryCLIService extends ThriftBinaryCLIService { public FakeEmbeddedThriftBinaryCLIService(HiveConf hiveConf) { - super(new CLIService()); + super(new CLIService(null)); isEmbedded = true; cliService.init(hiveConf); cliService.start(); diff --git a/service/src/test/org/apache/hive/service/server/TestServerOptionsProcessor.java b/service/src/test/org/apache/hive/service/server/TestServerOptionsProcessor.java index 1aea0b8..4b1d25e 100644 --- a/service/src/test/org/apache/hive/service/server/TestServerOptionsProcessor.java +++ b/service/src/test/org/apache/hive/service/server/TestServerOptionsProcessor.java @@ -21,6 +21,8 @@ import org.junit.Assert; import org.junit.Test; +import org.apache.hive.service.server.HiveServer2.ServerOptionsProcessor; + /** * Test ServerOptionsProcessor * @@ -39,17 +41,12 @@ public void test() { null, System.getProperty(key)); + optProcessor.parse(args); - boolean isSuccess = optProcessor.process(args); - Assert.assertTrue("options processor result", isSuccess); Assert.assertEquals( "checking system property after processing options", value, System.getProperty(key)); - - - - } } diff --git a/shims/0.20/pom.xml b/shims/0.20/pom.xml index 69eb151..0f6eb17 100644 --- a/shims/0.20/pom.xml +++ b/shims/0.20/pom.xml @@ -19,7 +19,7 @@ org.apache.hive hive - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../../pom.xml diff --git a/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java b/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java index a353a46..9d2ff55 100644 --- a/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java +++ b/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java @@ -619,6 +619,12 @@ public UserGroupInformation loginUserFromKeytabAndReturnUGI( } @Override + public String getResolvedPrincipal(String principal) throws IOException { + // Not supported + return null; + } + + @Override public void reLoginUserFromKeytab() throws IOException{ throwKerberosUnsupportedError(); } @@ -698,7 +704,7 @@ public void setFullFileStatus(Configuration conf, HdfsFileStatus sourceStatus, } public class Hadoop20FileStatus implements HdfsFileStatus { - private FileStatus fileStatus; + private final FileStatus fileStatus; public Hadoop20FileStatus(FileStatus fileStatus) { this.fileStatus = fileStatus; } @@ -706,6 +712,7 @@ public Hadoop20FileStatus(FileStatus fileStatus) { public FileStatus getFileStatus() { return fileStatus; } + @Override public void debugLog() { if (fileStatus != null) { LOG.debug(fileStatus.toString()); @@ -824,6 +831,11 @@ public short getDefaultReplication(FileSystem fs, Path path) { } @Override + public void refreshDefaultQueue(Configuration conf, String userName) { + // MR1 does not expose API required to set MR queue mapping for user + } + + @Override public String getTokenFileLocEnvName() { throw new UnsupportedOperationException( "Kerberos not supported in current hadoop version"); @@ -928,4 +940,15 @@ public boolean supportTrashFeature() { public Path getCurrentTrashPath(Configuration conf, FileSystem fs) { return null; } + + @Override + public KerberosNameShim getKerberosNameShim(String name) throws IOException { + // Not supported + return null; + } + + @Override + public void setZookeeperClientKerberosJaasConfig(String principal, String keyTabFile) { + // Not supported + } } diff --git a/shims/0.20S/pom.xml b/shims/0.20S/pom.xml index 4d184b0..6eb1417 100644 --- a/shims/0.20S/pom.xml +++ b/shims/0.20S/pom.xml @@ -19,7 +19,7 @@ org.apache.hive hive - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../../pom.xml diff --git a/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java b/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java index 030cb75..2626af7 100644 --- a/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java +++ b/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java @@ -60,6 +60,7 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.TaskAttemptID; import org.apache.hadoop.mapreduce.TaskID; +import org.apache.hadoop.security.KerberosName; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.VersionInfo; @@ -158,6 +159,11 @@ public short getDefaultReplication(FileSystem fs, Path path) { } @Override + public void refreshDefaultQueue(Configuration conf, String userName) { + // MR1 does not expose API required to set MR queue mapping for user + } + + @Override public void setTotalOrderPartitionFile(JobConf jobConf, Path partitionFile){ TotalOrderPartitioner.setPartitionFile(jobConf, partitionFile); } @@ -546,4 +552,44 @@ public boolean supportTrashFeature() { public Path getCurrentTrashPath(Configuration conf, FileSystem fs) { return null; } + + /** + * Returns a shim to wrap KerberosName + */ + @Override + public KerberosNameShim getKerberosNameShim(String name) throws IOException { + return new KerberosNameShim(name); + } + + /** + * Shim for KerberosName + */ + public class KerberosNameShim implements HadoopShimsSecure.KerberosNameShim { + + private KerberosName kerberosName; + + public KerberosNameShim(String name) { + kerberosName = new KerberosName(name); + } + + public String getDefaultRealm() { + return kerberosName.getDefaultRealm(); + } + + public String getServiceName() { + return kerberosName.getServiceName(); + } + + public String getHostName() { + return kerberosName.getHostName(); + } + + public String getRealm() { + return kerberosName.getRealm(); + } + + public String getShortName() throws IOException { + return kerberosName.getShortName(); + } + } } diff --git a/shims/0.23/pom.xml b/shims/0.23/pom.xml index ad71fcc..9af86c3 100644 --- a/shims/0.23/pom.xml +++ b/shims/0.23/pom.xml @@ -19,7 +19,7 @@ org.apache.hive hive - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../../pom.xml @@ -120,6 +120,17 @@ true + org.apache.hadoop + hadoop-yarn-server-resourcemanager + ${hadoop-23.version} + + + javax.servlet + servlet-api + + + + org.apache.tez tez-tests ${tez.version} diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java index 0731108..6125714 100644 --- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java +++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java @@ -72,8 +72,13 @@ import org.apache.hadoop.mapreduce.task.JobContextImpl; import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.authentication.util.KerberosName; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Progressable; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueuePlacementPolicy; import org.apache.tez.test.MiniTezCluster; import com.google.common.base.Joiner; @@ -85,6 +90,7 @@ * Implemention of shims against Hadoop 0.23.0. */ public class Hadoop23Shims extends HadoopShimsSecure { + private static final String MR2_JOB_QUEUE_PROPERTY = "mapreduce.job.queuename"; HadoopShims.MiniDFSShim cluster = null; @@ -220,6 +226,30 @@ public int compare(LongWritable o1, LongWritable o2) { } /** + * Load the fair scheduler queue for given user if available. + */ + @Override + public void refreshDefaultQueue(Configuration conf, String userName) throws IOException { + String requestedQueue = YarnConfiguration.DEFAULT_QUEUE_NAME; + if (StringUtils.isNotBlank(userName) && isFairScheduler(conf)) { + AllocationConfiguration allocConf = new AllocationConfiguration(conf); + QueuePlacementPolicy queuePolicy = allocConf.getPlacementPolicy(); + if (queuePolicy != null) { + requestedQueue = queuePolicy.assignAppToQueue(requestedQueue, userName); + if (StringUtils.isNotBlank(requestedQueue)) { + LOG.debug("Setting queue name to " + requestedQueue + " for user " + userName); + conf.set(MR2_JOB_QUEUE_PROPERTY, requestedQueue); + } + } + } + } + + private boolean isFairScheduler (Configuration conf) { + return FairScheduler.class.getName(). + equalsIgnoreCase(conf.get(YarnConfiguration.RM_SCHEDULER)); + } + + /** * Returns a shim to wrap MiniMrCluster */ @Override @@ -847,4 +877,44 @@ public Path getCurrentTrashPath(Configuration conf, FileSystem fs) { TrashPolicy tp = TrashPolicy.getInstance(conf, fs, fs.getHomeDirectory()); return tp.getCurrentTrashDir(); } + + /** + * Returns a shim to wrap KerberosName + */ + @Override + public KerberosNameShim getKerberosNameShim(String name) throws IOException { + return new KerberosNameShim(name); + } + + /** + * Shim for KerberosName + */ + public class KerberosNameShim implements HadoopShimsSecure.KerberosNameShim { + + private KerberosName kerberosName; + + public KerberosNameShim(String name) { + kerberosName = new KerberosName(name); + } + + public String getDefaultRealm() { + return kerberosName.getDefaultRealm(); + } + + public String getServiceName() { + return kerberosName.getServiceName(); + } + + public String getHostName() { + return kerberosName.getHostName(); + } + + public String getRealm() { + return kerberosName.getRealm(); + } + + public String getShortName() throws IOException { + return kerberosName.getShortName(); + } + } } diff --git a/shims/aggregator/pom.xml b/shims/aggregator/pom.xml index fe18ccb..5f76e84 100644 --- a/shims/aggregator/pom.xml +++ b/shims/aggregator/pom.xml @@ -19,7 +19,7 @@ org.apache.hive hive - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../../pom.xml diff --git a/shims/common-secure/pom.xml b/shims/common-secure/pom.xml index 68ac80b..2c56f5d 100644 --- a/shims/common-secure/pom.xml +++ b/shims/common-secure/pom.xml @@ -19,7 +19,7 @@ org.apache.hive hive - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../../pom.xml @@ -74,6 +74,11 @@ ${libthrift.version} + org.apache.curator + curator-framework + ${curator.version} + + org.apache.zookeeper zookeeper ${zookeeper.version} diff --git a/shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java b/shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java index 6e1ad03..53341e0 100644 --- a/shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java +++ b/shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java @@ -29,11 +29,14 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; -import javax.security.auth.login.LoginException; +import javax.security.auth.login.AppConfigurationEntry; +import javax.security.auth.login.AppConfigurationEntry.LoginModuleControlFlag; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.logging.Log; @@ -66,6 +69,7 @@ import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.util.KerberosUtil; import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; @@ -73,6 +77,7 @@ import org.apache.hadoop.tools.HadoopArchives; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.ToolRunner; +import org.apache.zookeeper.client.ZooKeeperSaslClient; import com.google.common.primitives.Longs; @@ -88,6 +93,7 @@ public String unquoteHtmlChars(String item) { return HtmlQuoting.unquoteHtmlChars(item); } + @Override public HadoopShims.CombineFileInputFormatShim getCombineFileInputFormat() { return new CombineFileInputFormatShim() { @Override @@ -171,6 +177,7 @@ public void write(DataOutput out) throws IOException { protected boolean isShrinked; protected long shrinkedLength; + @Override public boolean next(K key, V value) throws IOException { while ((curReader == null) @@ -183,11 +190,13 @@ public boolean next(K key, V value) throws IOException { return true; } + @Override public K createKey() { K newKey = curReader.createKey(); return (K)(new CombineHiveKey(newKey)); } + @Override public V createValue() { return curReader.createValue(); } @@ -195,10 +204,12 @@ public V createValue() { /** * Return the amount of data processed. */ + @Override public long getPos() throws IOException { return progress; } + @Override public void close() throws IOException { if (curReader != null) { curReader.close(); @@ -209,6 +220,7 @@ public void close() throws IOException { /** * Return progress based on the amount of data processed so far. */ + @Override public float getProgress() throws IOException { return Math.min(1.0f, progress / (float) (split.getLength())); } @@ -309,6 +321,7 @@ protected boolean initNextRecordReader(K key) throws IOException { CombineFileInputFormat implements HadoopShims.CombineFileInputFormatShim { + @Override public Path[] getInputPathsShim(JobConf conf) { try { return FileInputFormat.getInputPaths(conf); @@ -339,7 +352,7 @@ public void createPool(JobConf conf, PathFilter... filters) { super.setMaxSplitSize(minSize); } - InputSplit[] splits = (InputSplit[]) super.getSplits(job, numSplits); + InputSplit[] splits = super.getSplits(job, numSplits); ArrayList inputSplitShims = new ArrayList(); for (int pos = 0; pos < splits.length; pos++) { @@ -359,10 +372,12 @@ public void createPool(JobConf conf, PathFilter... filters) { return inputSplitShims.toArray(new InputSplitShim[inputSplitShims.size()]); } + @Override public InputSplitShim getInputSplitShim() throws IOException { return new InputSplitShim(); } + @Override public RecordReader getRecordReader(JobConf job, HadoopShims.InputSplitShim split, Reporter reporter, Class> rrClass) @@ -373,6 +388,7 @@ public RecordReader getRecordReader(JobConf job, HadoopShims.InputSplitShim spli } + @Override public String getInputFormatClassName() { return "org.apache.hadoop.hive.ql.io.CombineHiveInputFormat"; } @@ -401,6 +417,7 @@ public int createHadoopArchive(Configuration conf, Path sourceDir, Path destDir, * the archive as compared to the full path in case of earlier versions. * See this api in Hadoop20Shims for comparison. */ + @Override public URI getHarUri(URI original, URI base, URI originalBase) throws URISyntaxException { URI relative = originalBase.relativize(original); @@ -431,6 +448,7 @@ public void commitTask(TaskAttemptContext taskContext) { } public void abortTask(TaskAttemptContext taskContext) { } } + @Override public void prepareJobOutput(JobConf conf) { conf.setOutputCommitter(NullOutputCommitter.class); @@ -573,6 +591,17 @@ public UserGroupInformation loginUserFromKeytabAndReturnUGI( return UserGroupInformation.loginUserFromKeytabAndReturnUGI(hostPrincipal, keytabFile); } + /** + * Convert Kerberos principal name pattern to valid Kerberos principal names. + * @param principal (principal name pattern) + * @return + * @throws IOException + */ + @Override + public String getResolvedPrincipal(String principal) throws IOException { + return SecurityUtil.getServerPrincipal(principal, "0.0.0.0"); + } + @Override public String getTokenFileLocEnvName() { return UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION; @@ -675,4 +704,58 @@ public void checkFileAccess(FileSystem fs, FileStatus stat, FsAction action) throws IOException, AccessControlException, Exception { DefaultFileAccess.checkFileAccess(fs, stat, action); } + + @Override + public void setZookeeperClientKerberosJaasConfig(String principal, String keyTabFile) throws IOException { + // ZooKeeper property name to pick the correct JAAS conf section + final String SASL_LOGIN_CONTEXT_NAME = "HiveZooKeeperClient"; + System.setProperty(ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY, SASL_LOGIN_CONTEXT_NAME); + + principal = getResolvedPrincipal(principal); + JaasConfiguration jaasConf = new JaasConfiguration(SASL_LOGIN_CONTEXT_NAME, principal, keyTabFile); + + // Install the Configuration in the runtime. + javax.security.auth.login.Configuration.setConfiguration(jaasConf); + } + + /** + * A JAAS configuration for ZooKeeper clients intended to use for SASL + * Kerberos. + */ + private static class JaasConfiguration extends javax.security.auth.login.Configuration { + // Current installed Configuration + private final javax.security.auth.login.Configuration baseConfig = javax.security.auth.login.Configuration + .getConfiguration(); + private final String loginContextName; + private final String principal; + private final String keyTabFile; + + public JaasConfiguration(String hiveLoginContextName, String principal, String keyTabFile) { + this.loginContextName = hiveLoginContextName; + this.principal = principal; + this.keyTabFile = keyTabFile; + } + + @Override + public AppConfigurationEntry[] getAppConfigurationEntry(String appName) { + if (loginContextName.equals(appName)) { + Map krbOptions = new HashMap(); + krbOptions.put("doNotPrompt", "true"); + krbOptions.put("storeKey", "true"); + krbOptions.put("useKeyTab", "true"); + krbOptions.put("principal", principal); + krbOptions.put("keyTab", keyTabFile); + krbOptions.put("refreshKrb5Config", "true"); + AppConfigurationEntry hiveZooKeeperClientEntry = new AppConfigurationEntry( + KerberosUtil.getKrb5LoginModuleName(), LoginModuleControlFlag.REQUIRED, krbOptions); + return new AppConfigurationEntry[] { hiveZooKeeperClientEntry }; + } + // Try the base config + if (baseConfig != null) { + return baseConfig.getAppConfigurationEntry(appName); + } + return null; + } + } + } diff --git a/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/DBTokenStore.java b/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/DBTokenStore.java index b507a09..d86c5cb 100644 --- a/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/DBTokenStore.java +++ b/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/DBTokenStore.java @@ -25,6 +25,7 @@ import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge.Server.ServerMode; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager.DelegationTokenInformation; import org.apache.hadoop.security.token.delegation.HiveDelegationTokenSupport; @@ -108,18 +109,17 @@ public boolean removeToken(DelegationTokenIdentifier tokenIdentifier) throws Tok return delTokenIdents; } - private Object hmsHandler; + private Object rawStore; @Override - public void setStore(Object hms) throws TokenStoreException { - hmsHandler = hms; + public void init(Object rawStore, ServerMode smode) throws TokenStoreException { + this.rawStore = rawStore; } private Object invokeOnRawStore(String methName, Object[] params, Class ... paramTypes) throws TokenStoreException{ try { - Object rawStore = hmsHandler.getClass().getMethod("getMS").invoke(hmsHandler); return rawStore.getClass().getMethod(methName, paramTypes).invoke(rawStore, params); } catch (IllegalArgumentException e) { throw new TokenStoreException(e); @@ -149,5 +149,4 @@ public void close() throws IOException { // No-op. } - } diff --git a/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/DelegationTokenStore.java b/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/DelegationTokenStore.java index f3c2e48..867b4ed 100644 --- a/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/DelegationTokenStore.java +++ b/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/DelegationTokenStore.java @@ -21,6 +21,7 @@ import java.util.List; import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge.Server.ServerMode; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager.DelegationTokenInformation; /** @@ -108,6 +109,10 @@ DelegationTokenInformation getToken(DelegationTokenIdentifier tokenIdentifier) */ List getAllDelegationTokenIdentifiers() throws TokenStoreException; - void setStore(Object hmsHandler) throws TokenStoreException; + /** + * @param hmsHandler ObjectStore used by DBTokenStore + * @param smode Indicate whether this is a metastore or hiveserver2 token store + */ + void init(Object hmsHandler, ServerMode smode); } diff --git a/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge20S.java b/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge20S.java index 8b30b49..624ac6b 100644 --- a/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge20S.java +++ b/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge20S.java @@ -308,6 +308,10 @@ static String encodeIdentifier(byte[] identifier) { "hive.cluster.delegation.token.store.class"; public static final String DELEGATION_TOKEN_STORE_ZK_CONNECT_STR = "hive.cluster.delegation.token.store.zookeeper.connectString"; + // alternate connect string specification configuration + public static final String DELEGATION_TOKEN_STORE_ZK_CONNECT_STR_ALTERNATE = + "hive.zookeeper.quorum"; + public static final String DELEGATION_TOKEN_STORE_ZK_CONNECT_TIMEOUTMILLIS = "hive.cluster.delegation.token.store.zookeeper.connectTimeoutMillis"; public static final String DELEGATION_TOKEN_STORE_ZK_ZNODE = @@ -315,7 +319,7 @@ static String encodeIdentifier(byte[] identifier) { public static final String DELEGATION_TOKEN_STORE_ZK_ACL = "hive.cluster.delegation.token.store.zookeeper.acl"; public static final String DELEGATION_TOKEN_STORE_ZK_ZNODE_DEFAULT = - "/hive/cluster/delegation"; + "/hivedelegation"; public Server() throws TTransportException { try { @@ -417,7 +421,7 @@ protected DelegationTokenStore getTokenStore(Configuration conf) } @Override - public void startDelegationTokenSecretManager(Configuration conf, Object hms) + public void startDelegationTokenSecretManager(Configuration conf, Object rawStore, ServerMode smode) throws IOException{ long secretKeyInterval = conf.getLong(DELEGATION_KEY_UPDATE_INTERVAL_KEY, @@ -430,7 +434,7 @@ public void startDelegationTokenSecretManager(Configuration conf, Object hms) DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT); DelegationTokenStore dts = getTokenStore(conf); - dts.setStore(hms); + dts.init(rawStore, smode); secretManager = new TokenStoreDelegationTokenSecretManager(secretKeyInterval, tokenMaxLifetime, tokenRenewInterval, diff --git a/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/MemoryTokenStore.java b/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/MemoryTokenStore.java index 9908aa4..cf60b7c 100644 --- a/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/MemoryTokenStore.java +++ b/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/MemoryTokenStore.java @@ -26,6 +26,7 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge.Server.ServerMode; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager.DelegationTokenInformation; /** @@ -108,8 +109,7 @@ public void close() throws IOException { } @Override - public void setStore(Object hmsHandler) throws TokenStoreException { + public void init(Object hmsHandler, ServerMode smode) throws TokenStoreException { // no-op } - } diff --git a/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/TokenStoreDelegationTokenSecretManager.java b/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/TokenStoreDelegationTokenSecretManager.java index 4ccf895..8146d51 100644 --- a/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/TokenStoreDelegationTokenSecretManager.java +++ b/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/TokenStoreDelegationTokenSecretManager.java @@ -265,6 +265,7 @@ protected void removeExpiredTokens() { /** * Extension of rollMasterKey to remove expired keys from store. + * * @throws IOException */ protected void rollMasterKeyExt() throws IOException { @@ -273,18 +274,21 @@ protected void rollMasterKeyExt() throws IOException { HiveDelegationTokenSupport.rollMasterKey(TokenStoreDelegationTokenSecretManager.this); List keysAfterRoll = Arrays.asList(getAllKeys()); for (DelegationKey key : keysAfterRoll) { - keys.remove(key.getKeyId()); - if (key.getKeyId() == currentKeyId) { - tokenStore.updateMasterKey(currentKeyId, encodeWritable(key)); - } + keys.remove(key.getKeyId()); + if (key.getKeyId() == currentKeyId) { + tokenStore.updateMasterKey(currentKeyId, encodeWritable(key)); + } } for (DelegationKey expiredKey : keys.values()) { LOGGER.info("Removing expired key id={}", expiredKey.getKeyId()); - tokenStore.removeMasterKey(expiredKey.getKeyId()); + try { + tokenStore.removeMasterKey(expiredKey.getKeyId()); + } catch (Exception e) { + LOGGER.error("Error removing expired key id={}", expiredKey.getKeyId(), e); + } } } - /** * Cloned from {@link AbstractDelegationTokenSecretManager} to deal with private access * restriction (there would not be an need to clone the remove thread if the remove logic was diff --git a/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/ZooKeeperTokenStore.java b/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/ZooKeeperTokenStore.java index 8683496..16a52e4 100644 --- a/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/ZooKeeperTokenStore.java +++ b/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/ZooKeeperTokenStore.java @@ -20,24 +20,28 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; import org.apache.commons.lang.StringUtils; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.framework.api.ACLProvider; +import org.apache.curator.framework.imps.CuratorFrameworkState; +import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.shims.ShimLoader; +import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge.Server.ServerMode; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager.DelegationTokenInformation; import org.apache.hadoop.security.token.delegation.HiveDelegationTokenSupport; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.Watcher; import org.apache.zookeeper.ZooDefs; import org.apache.zookeeper.ZooDefs.Ids; -import org.apache.zookeeper.ZooKeeper; -import org.apache.zookeeper.ZooKeeper.States; +import org.apache.zookeeper.ZooDefs.Perms; import org.apache.zookeeper.data.ACL; import org.apache.zookeeper.data.Id; import org.slf4j.Logger; @@ -56,26 +60,35 @@ private static final String NODE_TOKENS = "/tokens"; private String rootNode = ""; - private volatile ZooKeeper zkSession; + private volatile CuratorFramework zkSession; private String zkConnectString; private final int zkSessionTimeout = 3000; - private long connectTimeoutMillis = -1; - private List newNodeAcl = Ids.OPEN_ACL_UNSAFE; - - private class ZooKeeperWatcher implements Watcher { - public void process(org.apache.zookeeper.WatchedEvent event) { - LOGGER.info(event.toString()); - if (event.getState() == Watcher.Event.KeeperState.Expired) { - LOGGER.warn("ZooKeeper session expired, discarding connection"); - try { - zkSession.close(); - } catch (Throwable e) { - LOGGER.warn("Failed to close connection on expired session", e); - } - } + private int connectTimeoutMillis = -1; + private List newNodeAcl = Arrays.asList(new ACL(Perms.ALL, Ids.AUTH_IDS)); + + /** + * ACLProvider permissions will be used in case parent dirs need to be created + */ + private final ACLProvider aclDefaultProvider = new ACLProvider() { + + @Override + public List getDefaultAcl() { + return newNodeAcl; } - } + @Override + public List getAclForPath(String path) { + return getDefaultAcl(); + } + }; + + + private ServerMode serverMode; + + private final String WHEN_ZK_DSTORE_MSG = "when zookeeper based delegation token storage is enabled" + + "(hive.cluster.delegation.token.store.class=" + ZooKeeperTokenStore.class.getName() + ")"; + + private Configuration conf; /** * Default constructor for dynamic instantiation w/ Configurable @@ -84,93 +97,74 @@ public void process(org.apache.zookeeper.WatchedEvent event) { protected ZooKeeperTokenStore() { } - public ZooKeeperTokenStore(String hostPort) { - this.zkConnectString = hostPort; - init(); - } - - private ZooKeeper getSession() { - if (zkSession == null || zkSession.getState() == States.CLOSED) { - synchronized (this) { - if (zkSession == null || zkSession.getState() == States.CLOSED) { - try { - zkSession = createConnectedClient(this.zkConnectString, this.zkSessionTimeout, - this.connectTimeoutMillis, new ZooKeeperWatcher()); - } catch (IOException ex) { - throw new TokenStoreException("Token store error.", ex); - } - } + private CuratorFramework getSession() { + if (zkSession == null || zkSession.getState() == CuratorFrameworkState.STOPPED) { + synchronized (this) { + if (zkSession == null || zkSession.getState() == CuratorFrameworkState.STOPPED) { + zkSession = CuratorFrameworkFactory.builder().connectString(zkConnectString) + .sessionTimeoutMs(zkSessionTimeout).connectionTimeoutMs(connectTimeoutMillis) + .aclProvider(aclDefaultProvider) + .retryPolicy(new ExponentialBackoffRetry(1000, 3)).build(); + zkSession.start(); } + } } return zkSession; } - /** - * Create a ZooKeeper session that is in connected state. - * - * @param connectString ZooKeeper connect String - * @param sessionTimeout ZooKeeper session timeout - * @param connectTimeout milliseconds to wait for connection, 0 or negative value means no wait - * @param watchers - * @return - * @throws InterruptedException - * @throws IOException - */ - public static ZooKeeper createConnectedClient(String connectString, - int sessionTimeout, long connectTimeout, final Watcher... watchers) - throws IOException { - final CountDownLatch connected = new CountDownLatch(1); - Watcher connectWatcher = new Watcher() { - @Override - public void process(WatchedEvent event) { - switch (event.getState()) { - case SyncConnected: - connected.countDown(); - break; - } - for (Watcher w : watchers) { - w.process(event); - } - } - }; - ZooKeeper zk = new ZooKeeper(connectString, sessionTimeout, connectWatcher); - if (connectTimeout > 0) { - try { - if (!connected.await(connectTimeout, TimeUnit.MILLISECONDS)) { - zk.close(); - throw new IOException("Timeout waiting for connection after " - + connectTimeout + "ms"); - } - } catch (InterruptedException e) { - throw new IOException("Error waiting for connection.", e); - } + private void setupJAASConfig(Configuration conf) throws IOException { + if (!UserGroupInformation.getLoginUser().isFromKeytab()) { + // The process has not logged in using keytab + // this should be a test mode, can't use keytab to authenticate + // with zookeeper. + LOGGER.warn("Login is not from keytab"); + return; } - return zk; + + String principal; + String keytab; + switch (serverMode) { + case METASTORE: + principal = getNonEmptyConfVar(conf, "hive.metastore.kerberos.principal"); + keytab = getNonEmptyConfVar(conf, "hive.metastore.kerberos.keytab.file"); + break; + case HIVESERVER2: + principal = getNonEmptyConfVar(conf, "hive.server2.authentication.kerberos.principal"); + keytab = getNonEmptyConfVar(conf, "hive.server2.authentication.kerberos.keytab"); + break; + default: + throw new AssertionError("Unexpected server mode " + serverMode); + } + ShimLoader.getHadoopShims().setZookeeperClientKerberosJaasConfig(principal, keytab); + } + + private String getNonEmptyConfVar(Configuration conf, String param) throws IOException { + String val = conf.get(param); + if (val == null || val.trim().isEmpty()) { + throw new IOException("Configuration parameter " + param + " should be set, " + + WHEN_ZK_DSTORE_MSG); + } + return val; } /** * Create a path if it does not already exist ("mkdir -p") - * @param zk ZooKeeper session * @param path string with '/' separator * @param acl list of ACL entries - * @return - * @throws KeeperException - * @throws InterruptedException + * @throws TokenStoreException */ - public static String ensurePath(ZooKeeper zk, String path, List acl) throws KeeperException, - InterruptedException { - String[] pathComps = StringUtils.splitByWholeSeparator(path, "/"); - String currentPath = ""; - for (String pathComp : pathComps) { - currentPath += "/" + pathComp; - try { - String node = zk.create(currentPath, new byte[0], acl, - CreateMode.PERSISTENT); - LOGGER.info("Created path: " + node); - } catch (KeeperException.NodeExistsException e) { - } + public void ensurePath(String path, List acl) + throws TokenStoreException { + try { + CuratorFramework zk = getSession(); + String node = zk.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT) + .withACL(acl).forPath(path); + LOGGER.info("Created path: {} ", node); + } catch (KeeperException.NodeExistsException e) { + // node already exists + } catch (Exception e) { + throw new TokenStoreException("Error creating path " + path, e); } - return currentPath; } /** @@ -234,45 +228,24 @@ public static int getPermFromString(String permString) { return acl; } - private void init() { - if (this.zkConnectString == null) { - throw new IllegalStateException("Not initialized"); - } - + private void initClientAndPaths() { if (this.zkSession != null) { - try { - this.zkSession.close(); - } catch (InterruptedException ex) { - LOGGER.warn("Failed to close existing session.", ex); - } + this.zkSession.close(); } - ZooKeeper zk = getSession(); - try { - ensurePath(zk, rootNode + NODE_KEYS, newNodeAcl); - ensurePath(zk, rootNode + NODE_TOKENS, newNodeAcl); - } catch (Exception e) { - throw new TokenStoreException("Failed to validate token path.", e); - } + ensurePath(rootNode + NODE_KEYS, newNodeAcl); + ensurePath(rootNode + NODE_TOKENS, newNodeAcl); + } catch (TokenStoreException e) { + throw e; + } } @Override public void setConf(Configuration conf) { if (conf == null) { - throw new IllegalArgumentException("conf is null"); + throw new IllegalArgumentException("conf is null"); } - this.zkConnectString = conf.get( - HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_CONNECT_STR, null); - this.connectTimeoutMillis = conf.getLong( - HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_CONNECT_TIMEOUTMILLIS, -1); - this.rootNode = conf.get( - HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_ZNODE, - HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_ZNODE_DEFAULT); - String csv = conf.get(HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_ACL, null); - if (StringUtils.isNotBlank(csv)) { - this.newNodeAcl = parseACLs(csv); - } - init(); + this.conf = conf; } @Override @@ -280,15 +253,18 @@ public Configuration getConf() { return null; // not required } - private Map getAllKeys() throws KeeperException, - InterruptedException { + private Map getAllKeys() throws KeeperException, InterruptedException { String masterKeyNode = rootNode + NODE_KEYS; - ZooKeeper zk = getSession(); - List nodes = zk.getChildren(masterKeyNode, false); + + // get children of key node + List nodes = zkGetChildren(masterKeyNode); + + // read each child node, add to results Map result = new HashMap(); for (String node : nodes) { - byte[] data = zk.getData(masterKeyNode + "/" + node, false, null); + String nodePath = masterKeyNode + "/" + node; + byte[] data = zkGetData(nodePath); if (data != null) { result.put(getSeq(node), data); } @@ -296,6 +272,26 @@ public Configuration getConf() { return result; } + private List zkGetChildren(String path) { + CuratorFramework zk = getSession(); + try { + return zk.getChildren().forPath(path); + } catch (Exception e) { + throw new TokenStoreException("Error getting children for " + path, e); + } + } + + private byte[] zkGetData(String nodePath) { + CuratorFramework zk = getSession(); + try { + return zk.getData().forPath(nodePath); + } catch (KeeperException.NoNodeException ex) { + return null; + } catch (Exception e) { + throw new TokenStoreException("Error reading " + nodePath, e); + } + } + private int getSeq(String path) { String[] pathComps = path.split("/"); return Integer.parseInt(pathComps[pathComps.length-1]); @@ -303,44 +299,45 @@ private int getSeq(String path) { @Override public int addMasterKey(String s) { + String keysPath = rootNode + NODE_KEYS + "/"; + CuratorFramework zk = getSession(); + String newNode; try { - ZooKeeper zk = getSession(); - String newNode = zk.create(rootNode + NODE_KEYS + "/", s.getBytes(), newNodeAcl, - CreateMode.PERSISTENT_SEQUENTIAL); - LOGGER.info("Added key {}", newNode); - return getSeq(newNode); - } catch (KeeperException ex) { - throw new TokenStoreException(ex); - } catch (InterruptedException ex) { - throw new TokenStoreException(ex); + newNode = zk.create().withMode(CreateMode.PERSISTENT_SEQUENTIAL).withACL(newNodeAcl) + .forPath(keysPath, s.getBytes()); + } catch (Exception e) { + throw new TokenStoreException("Error creating new node with path " + keysPath, e); } + LOGGER.info("Added key {}", newNode); + return getSeq(newNode); } @Override public void updateMasterKey(int keySeq, String s) { + CuratorFramework zk = getSession(); + String keyPath = rootNode + NODE_KEYS + "/" + String.format(ZK_SEQ_FORMAT, keySeq); try { - ZooKeeper zk = getSession(); - zk.setData(rootNode + NODE_KEYS + "/" + String.format(ZK_SEQ_FORMAT, keySeq), s.getBytes(), - -1); - } catch (KeeperException ex) { - throw new TokenStoreException(ex); - } catch (InterruptedException ex) { - throw new TokenStoreException(ex); + zk.setData().forPath(keyPath, s.getBytes()); + } catch (Exception e) { + throw new TokenStoreException("Error setting data in " + keyPath, e); } } @Override public boolean removeMasterKey(int keySeq) { + String keyPath = rootNode + NODE_KEYS + "/" + String.format(ZK_SEQ_FORMAT, keySeq); + zkDelete(keyPath); + return true; + } + + private void zkDelete(String path) { + CuratorFramework zk = getSession(); try { - ZooKeeper zk = getSession(); - zk.delete(rootNode + NODE_KEYS + "/" + String.format(ZK_SEQ_FORMAT, keySeq), -1); - return true; + zk.delete().forPath(path); } catch (KeeperException.NoNodeException ex) { - return false; - } catch (KeeperException ex) { - throw new TokenStoreException(ex); - } catch (InterruptedException ex) { - throw new TokenStoreException(ex); + // already deleted + } catch (Exception e) { + throw new TokenStoreException("Error deleting " + path, e); } } @@ -374,67 +371,42 @@ private String getTokenPath(DelegationTokenIdentifier tokenIdentifier) { @Override public boolean addToken(DelegationTokenIdentifier tokenIdentifier, DelegationTokenInformation token) { + byte[] tokenBytes = HiveDelegationTokenSupport.encodeDelegationTokenInformation(token); + String tokenPath = getTokenPath(tokenIdentifier); + CuratorFramework zk = getSession(); + String newNode; try { - ZooKeeper zk = getSession(); - byte[] tokenBytes = HiveDelegationTokenSupport.encodeDelegationTokenInformation(token); - String newNode = zk.create(getTokenPath(tokenIdentifier), - tokenBytes, newNodeAcl, CreateMode.PERSISTENT); - LOGGER.info("Added token: {}", newNode); - return true; - } catch (KeeperException.NodeExistsException ex) { - return false; - } catch (KeeperException ex) { - throw new TokenStoreException(ex); - } catch (InterruptedException ex) { - throw new TokenStoreException(ex); + newNode = zk.create().withMode(CreateMode.PERSISTENT).withACL(newNodeAcl) + .forPath(tokenPath, tokenBytes); + } catch (Exception e) { + throw new TokenStoreException("Error creating new node with path " + tokenPath, e); } + + LOGGER.info("Added token: {}", newNode); + return true; } @Override public boolean removeToken(DelegationTokenIdentifier tokenIdentifier) { - try { - ZooKeeper zk = getSession(); - zk.delete(getTokenPath(tokenIdentifier), -1); - return true; - } catch (KeeperException.NoNodeException ex) { - return false; - } catch (KeeperException ex) { - throw new TokenStoreException(ex); - } catch (InterruptedException ex) { - throw new TokenStoreException(ex); - } + String tokenPath = getTokenPath(tokenIdentifier); + zkDelete(tokenPath); + return true; } @Override public DelegationTokenInformation getToken(DelegationTokenIdentifier tokenIdentifier) { + byte[] tokenBytes = zkGetData(getTokenPath(tokenIdentifier)); try { - ZooKeeper zk = getSession(); - byte[] tokenBytes = zk.getData(getTokenPath(tokenIdentifier), false, null); - try { - return HiveDelegationTokenSupport.decodeDelegationTokenInformation(tokenBytes); - } catch (Exception ex) { - throw new TokenStoreException("Failed to decode token", ex); - } - } catch (KeeperException.NoNodeException ex) { - return null; - } catch (KeeperException ex) { - throw new TokenStoreException(ex); - } catch (InterruptedException ex) { - throw new TokenStoreException(ex); + return HiveDelegationTokenSupport.decodeDelegationTokenInformation(tokenBytes); + } catch (Exception ex) { + throw new TokenStoreException("Failed to decode token", ex); } } @Override public List getAllDelegationTokenIdentifiers() { String containerNode = rootNode + NODE_TOKENS; - final List nodes; - try { - nodes = getSession().getChildren(containerNode, false); - } catch (KeeperException ex) { - throw new TokenStoreException(ex); - } catch (InterruptedException ex) { - throw new TokenStoreException(ex); - } + final List nodes = zkGetChildren(containerNode); List result = new java.util.ArrayList( nodes.size()); for (String node : nodes) { @@ -452,17 +424,44 @@ public DelegationTokenInformation getToken(DelegationTokenIdentifier tokenIdenti @Override public void close() throws IOException { if (this.zkSession != null) { - try { - this.zkSession.close(); - } catch (InterruptedException ex) { - LOGGER.warn("Failed to close existing session.", ex); - } + this.zkSession.close(); } } @Override - public void setStore(Object hmsHandler) throws TokenStoreException { - // no-op. + public void init(Object objectStore, ServerMode smode) { + this.serverMode = smode; + zkConnectString = conf.get( + HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_CONNECT_STR, null); + if (zkConnectString == null || zkConnectString.trim().isEmpty()) { + // try alternate config param + zkConnectString = conf.get( + HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_CONNECT_STR_ALTERNATE, null); + if (zkConnectString == null || zkConnectString.trim().isEmpty()) { + throw new IllegalArgumentException("Zookeeper connect string has to be specifed through " + + "either " + HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_CONNECT_STR + + " or " + + HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_CONNECT_STR_ALTERNATE + + WHEN_ZK_DSTORE_MSG); + } + } + connectTimeoutMillis = conf.getInt( + HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_CONNECT_TIMEOUTMILLIS, -1); + String aclStr = conf.get(HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_ACL, null); + if (StringUtils.isNotBlank(aclStr)) { + this.newNodeAcl = parseACLs(aclStr); + } + rootNode = conf.get(HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_ZNODE, + HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_ZNODE_DEFAULT) + serverMode; + + try { + // Install the JAAS Configuration for the runtime + setupJAASConfig(conf); + } catch (IOException e) { + throw new TokenStoreException("Error setting up JAAS configuration for zookeeper client " + + e.getMessage(), e); + } + initClientAndPaths(); } } diff --git a/shims/common/pom.xml b/shims/common/pom.xml index 889b1ca..f8d779d 100644 --- a/shims/common/pom.xml +++ b/shims/common/pom.xml @@ -19,7 +19,7 @@ org.apache.hive hive - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../../pom.xml diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java index 4fcaa1e..f079cf8 100644 --- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java +++ b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java @@ -239,6 +239,15 @@ public URI getHarUri(URI original, URI base, URI originalBase) public String getTokenStrForm(String tokenSignature) throws IOException; /** + * Dynamically sets up the JAAS configuration that uses kerberos + * @param principal + * @param keyTabFile + * @throws IOException + */ + public void setZookeeperClientKerberosJaasConfig(String principal, String keyTabFile) + throws IOException; + + /** * Add a delegation token to the given ugi * @param ugi * @param tokenStr @@ -322,6 +331,14 @@ public UserGroupInformation loginUserFromKeytabAndReturnUGI(String principal, String keytabFile) throws IOException; /** + * Convert Kerberos principal name pattern to valid Kerberos principal names. + * @param principal (principal name pattern) + * @return + * @throws IOException + */ + public String getResolvedPrincipal(String principal) throws IOException; + + /** * Perform kerberos re-login using the given principal and keytab, to renew * the credentials * @throws IOException @@ -366,6 +383,15 @@ public boolean moveToAppropriateTrash(FileSystem fs, Path path, Configuration co public short getDefaultReplication(FileSystem fs, Path path); /** + * Reset the default fair scheduler queue mapping to end user. + * + * @param conf + * @param userName end user name + */ + public void refreshDefaultQueue(Configuration conf, String userName) + throws IOException; + + /** * Create the proxy ugi for the given userid * @param userName * @return @@ -731,4 +757,21 @@ public void checkFileAccess(FileSystem fs, FileStatus status, FsAction action) * @return Path to HDFS trash, if current hadoop supports trash feature. Null otherwise. */ Path getCurrentTrashPath(Configuration conf, FileSystem fs); + + /** + * Returns a shim to wrap KerberosName + */ + public KerberosNameShim getKerberosNameShim(String name) throws IOException; + + /** + * Shim for KerberosName + */ + public interface KerberosNameShim { + public String getDefaultRealm(); + public String getServiceName(); + public String getHostName(); + public String getRealm(); + public String getShortName() throws IOException; + } + } diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge.java b/shims/common/src/main/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge.java index d0d6c7b..d011c67 100644 --- a/shims/common/src/main/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge.java +++ b/shims/common/src/main/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge.java @@ -99,12 +99,15 @@ public abstract TTransport createClientTransport( } public static abstract class Server { + public enum ServerMode { + HIVESERVER2, METASTORE + }; public abstract TTransportFactory createTransportFactory(Map saslProps) throws TTransportException; public abstract TProcessor wrapProcessor(TProcessor processor); public abstract TProcessor wrapNonAssumingProcessor(TProcessor processor); public abstract InetAddress getRemoteAddress(); public abstract void startDelegationTokenSecretManager(Configuration conf, - Object hmsHandler) throws IOException; + Object hmsHandler, ServerMode smode) throws IOException; public abstract String getDelegationToken(String owner, String renewer) throws IOException, InterruptedException; public abstract String getDelegationTokenWithService(String owner, String renewer, String service) diff --git a/shims/pom.xml b/shims/pom.xml index f54621d..7027126 100644 --- a/shims/pom.xml +++ b/shims/pom.xml @@ -19,7 +19,7 @@ org.apache.hive hive - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../pom.xml diff --git a/spark-client/pom.xml b/spark-client/pom.xml index 2cc8c84..0842811 100644 --- a/spark-client/pom.xml +++ b/spark-client/pom.xml @@ -22,14 +22,14 @@ org.apache.hive hive - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT org.apache.hive spark-client jar Spark Remote Client - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT .. diff --git a/testutils/pom.xml b/testutils/pom.xml index b1ea4db..f5d84b2 100644 --- a/testutils/pom.xml +++ b/testutils/pom.xml @@ -19,7 +19,7 @@ org.apache.hive hive - 0.14.0-SNAPSHOT + 0.15.0-SNAPSHOT ../pom.xml diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/JIRAService.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/JIRAService.java index ca59ac4..88d8cab 100644 --- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/JIRAService.java +++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/JIRAService.java @@ -87,7 +87,7 @@ void postComment(boolean error, int numTestsExecuted, SortedSet failedTe List messages) { DefaultHttpClient httpClient = new DefaultHttpClient(); try { - String buildTag = formatBuildTag(mBuildTag); + BuildInfo buildInfo = formatBuildTag(mBuildTag); String buildTagForLogs = formatBuildTagForLogs(mBuildTag); List comments = Lists.newArrayList(); comments.add(""); @@ -120,8 +120,10 @@ void postComment(boolean error, int numTestsExecuted, SortedSet failedTe } comments.add(""); } - comments.add("Test results: " + mJenkinsURL + "/" + buildTag + "/testReport"); - comments.add("Console output: " + mJenkinsURL + "/" + buildTag + "/console"); + comments.add("Test results: " + mJenkinsURL + "/" + + buildInfo.getFormattedBuildTag() + "/testReport"); + comments.add("Console output: " + mJenkinsURL + "/" + + buildInfo.getFormattedBuildTag() + "/console"); comments.add("Test logs: " + mLogsURL + buildTagForLogs); comments.add(""); if(!messages.isEmpty()) { @@ -133,10 +135,9 @@ void postComment(boolean error, int numTestsExecuted, SortedSet failedTe } comments.add("This message is automatically generated."); String attachmentId = parseAttachementId(mPatch); - if(!attachmentId.isEmpty()) { - comments.add(""); - comments.add("ATTACHMENT ID: " + attachmentId); - } + comments.add(""); + comments.add("ATTACHMENT ID: " + attachmentId + + " - " + buildInfo.getBuildName()); mLogger.info("Comment: " + Joiner.on("\n").join(comments)); String body = Joiner.on("\n").join(comments); String url = String.format("%s/rest/api/2/issue/%s/comment", mUrl, mName); @@ -193,16 +194,36 @@ public void setBody(String body) { } } + + public static class BuildInfo { + private String buildName; + private String formattedBuildTag; + + public BuildInfo (String buildName, String formattedBuildTag) { + this.buildName = buildName; + this.formattedBuildTag = formattedBuildTag; + } + + public String getBuildName() { + return buildName; + } + + public String getFormattedBuildTag() { + return formattedBuildTag; + } + } + /** * Hive-Build-123 to Hive-Build/123 */ @VisibleForTesting - static String formatBuildTag(String buildTag) { + static BuildInfo formatBuildTag(String buildTag) { if(buildTag.contains("-")) { int lastDashIndex = buildTag.lastIndexOf("-"); String buildName = buildTag.substring(0, lastDashIndex); String buildId = buildTag.substring(lastDashIndex + 1); - return buildName + "/" + buildId; + String formattedBuildTag = buildName + "/" + buildId; + return new BuildInfo(buildName, formattedBuildTag); } throw new IllegalArgumentException("Build tag '" + buildTag + "' must contain a -"); } diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java index 3455137..696b790 100644 --- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java +++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java @@ -287,77 +287,77 @@ public static void main(String[] args) throws Exception { String buildTag = System.getenv("BUILD_TAG") == null ? "undefined-" + System.currentTimeMillis() : System.getenv("BUILD_TAG"); File logDir = Dirs.create(new File(executionContextConfiguration.getGlobalLogDirectory(), buildTag)); - LogDirectoryCleaner cleaner = new LogDirectoryCleaner(new File(executionContextConfiguration. - getGlobalLogDirectory()), 5); - cleaner.setName("LogCleaner-" + executionContextConfiguration.getGlobalLogDirectory()); - cleaner.setDaemon(true); - cleaner.start(); - TestConfiguration conf = TestConfiguration.fromFile(testConfigurationFile, LOG); - String repository = Strings.nullToEmpty(commandLine.getOptionValue(REPOSITORY)).trim(); - if(!repository.isEmpty()) { - conf.setRepository(repository); - } - String repositoryName = Strings.nullToEmpty(commandLine.getOptionValue(REPOSITORY_NAME)).trim(); - if(!repositoryName.isEmpty()) { - conf.setRepositoryName(repositoryName); - } - String branch = Strings.nullToEmpty(commandLine.getOptionValue(BRANCH)).trim(); - if(!branch.isEmpty()) { - conf.setBranch(branch); - } - String patch = Strings.nullToEmpty(commandLine.getOptionValue(PATCH)).trim(); - if(!patch.isEmpty()) { - conf.setPatch(patch); - } - String javaHome = Strings.nullToEmpty(commandLine.getOptionValue(JAVA_HOME)).trim(); - if(!javaHome.isEmpty()) { - conf.setJavaHome(javaHome); - } - String javaHomeForTests = Strings.nullToEmpty(commandLine.getOptionValue(JAVA_HOME_TEST)).trim(); - if(!javaHomeForTests.isEmpty()) { - conf.setJavaHomeForTests(javaHomeForTests); - } - String antTestArgs = Strings.nullToEmpty(commandLine.getOptionValue(ANT_TEST_ARGS)).trim(); - if(!antTestArgs.isEmpty()) { - conf.setAntTestArgs(antTestArgs); - } - String antEnvOpts = Strings.nullToEmpty(commandLine.getOptionValue(ANT_ENV_OPTS)).trim(); - if(!antEnvOpts.isEmpty()) { - conf.setAntEnvOpts(antEnvOpts); - } - String antTestTarget = Strings.nullToEmpty(commandLine.getOptionValue(ANT_TEST_TARGET)).trim(); - if(!antTestTarget.isEmpty()) { - conf.setAntTestTarget(antTestTarget); - } - String[] supplementalAntArgs = commandLine.getOptionValues(ANT_ARG); - if(supplementalAntArgs != null && supplementalAntArgs.length > 0) { - String antArgs = Strings.nullToEmpty(conf.getAntArgs()); - if(!(antArgs.isEmpty() || antArgs.endsWith(" "))) { - antArgs += " "; - } - antArgs += "-" + ANT_ARG + Joiner.on(" -" + ANT_ARG).join(supplementalAntArgs); - conf.setAntArgs(antArgs); - } - ExecutionContextProvider executionContextProvider = null; - ExecutionContext executionContext = null; - int exitCode = 0; - try { - executionContextProvider = executionContextConfiguration - .getExecutionContextProvider(); - executionContext = executionContextProvider.createExecutionContext(); - LocalCommandFactory localCommandFactory = new LocalCommandFactory(LOG); - PTest ptest = new PTest(conf, executionContext, buildTag, logDir, - localCommandFactory, new SSHCommandExecutor(LOG, localCommandFactory, conf.getSshOpts()), - new RSyncCommandExecutor(LOG, 10, localCommandFactory), LOG); - exitCode = ptest.run(); - } finally { - if(executionContext != null) { - executionContext.terminate(); - } - if(executionContextProvider != null) { - executionContextProvider.close(); - } - } - System.exit(exitCode); + LogDirectoryCleaner cleaner = new LogDirectoryCleaner(new File(executionContextConfiguration. + getGlobalLogDirectory()), 5); + cleaner.setName("LogCleaner-" + executionContextConfiguration.getGlobalLogDirectory()); + cleaner.setDaemon(true); + cleaner.start(); + TestConfiguration conf = TestConfiguration.fromFile(testConfigurationFile, LOG); + String repository = Strings.nullToEmpty(commandLine.getOptionValue(REPOSITORY)).trim(); + if(!repository.isEmpty()) { + conf.setRepository(repository); + } + String repositoryName = Strings.nullToEmpty(commandLine.getOptionValue(REPOSITORY_NAME)).trim(); + if(!repositoryName.isEmpty()) { + conf.setRepositoryName(repositoryName); + } + String branch = Strings.nullToEmpty(commandLine.getOptionValue(BRANCH)).trim(); + if(!branch.isEmpty()) { + conf.setBranch(branch); + } + String patch = Strings.nullToEmpty(commandLine.getOptionValue(PATCH)).trim(); + if(!patch.isEmpty()) { + conf.setPatch(patch); + } + String javaHome = Strings.nullToEmpty(commandLine.getOptionValue(JAVA_HOME)).trim(); + if(!javaHome.isEmpty()) { + conf.setJavaHome(javaHome); + } + String javaHomeForTests = Strings.nullToEmpty(commandLine.getOptionValue(JAVA_HOME_TEST)).trim(); + if(!javaHomeForTests.isEmpty()) { + conf.setJavaHomeForTests(javaHomeForTests); + } + String antTestArgs = Strings.nullToEmpty(commandLine.getOptionValue(ANT_TEST_ARGS)).trim(); + if(!antTestArgs.isEmpty()) { + conf.setAntTestArgs(antTestArgs); + } + String antEnvOpts = Strings.nullToEmpty(commandLine.getOptionValue(ANT_ENV_OPTS)).trim(); + if(!antEnvOpts.isEmpty()) { + conf.setAntEnvOpts(antEnvOpts); + } + String antTestTarget = Strings.nullToEmpty(commandLine.getOptionValue(ANT_TEST_TARGET)).trim(); + if(!antTestTarget.isEmpty()) { + conf.setAntTestTarget(antTestTarget); + } + String[] supplementalAntArgs = commandLine.getOptionValues(ANT_ARG); + if(supplementalAntArgs != null && supplementalAntArgs.length > 0) { + String antArgs = Strings.nullToEmpty(conf.getAntArgs()); + if(!(antArgs.isEmpty() || antArgs.endsWith(" "))) { + antArgs += " "; + } + antArgs += "-" + ANT_ARG + Joiner.on(" -" + ANT_ARG).join(supplementalAntArgs); + conf.setAntArgs(antArgs); + } + ExecutionContextProvider executionContextProvider = null; + ExecutionContext executionContext = null; + int exitCode = 0; + try { + executionContextProvider = executionContextConfiguration + .getExecutionContextProvider(); + executionContext = executionContextProvider.createExecutionContext(); + LocalCommandFactory localCommandFactory = new LocalCommandFactory(LOG); + PTest ptest = new PTest(conf, executionContext, buildTag, logDir, + localCommandFactory, new SSHCommandExecutor(LOG, localCommandFactory, conf.getSshOpts()), + new RSyncCommandExecutor(LOG, 10, localCommandFactory), LOG); + exitCode = ptest.run(); + } finally { + if(executionContext != null) { + executionContext.terminate(); + } + if(executionContextProvider != null) { + executionContextProvider.close(); + } + } + System.exit(exitCode); } } diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/ExecutionContext.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/ExecutionContext.java index 07aa89e..b09de1d 100644 --- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/ExecutionContext.java +++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/ExecutionContext.java @@ -46,7 +46,7 @@ public void addBadHost(Host host) { public void clearBadHosts() { mBadHosts.clear(); } - void addHost(Host host) { + public void addHost(Host host) { mHosts.add(host); } boolean removeHost(Host host) { diff --git a/testutils/ptest2/src/main/resources/batch-exec.vm b/testutils/ptest2/src/main/resources/batch-exec.vm index 7a81849..6289785 100644 --- a/testutils/ptest2/src/main/resources/batch-exec.vm +++ b/testutils/ptest2/src/main/resources/batch-exec.vm @@ -62,7 +62,7 @@ then testModule=./ fi pushd $testModule - timeout 2h mvn -B -o test -Dmaven.repo.local=$localDir/$instanceName/maven \ + timeout 2h mvn -B test -Dmaven.repo.local=$localDir/$instanceName/maven \ $mavenArgs $mavenTestArgs $testArguments 1>$logDir/maven-test.txt 2>&1 /some/log/dir/maven-test.txt 2>&1 /some/log/dir/maven-test.txt 2>&1