diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 9df6656..1e3bf89 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -301,6 +301,9 @@ HIVE_IN_TEST("hive.in.test", false, "internal usage only, true in test mode", true), + HIVE_IN_TEZ_TEST("hive.in.tez.test", false, "internal use only, true when in testing tez", + true), + LOCALMODEAUTO("hive.exec.mode.local.auto", false, "Let Hive determine whether to run in local mode automatically"), LOCALMODEMAXBYTES("hive.exec.mode.local.auto.inputbytes.max", 134217728L, diff --git data/conf/tez/hive-site.xml data/conf/tez/hive-site.xml index 5165b91..e0238aa 100644 --- data/conf/tez/hive-site.xml +++ data/conf/tez/hive-site.xml @@ -245,4 +245,13 @@ + + hive.in.tez.test + true + + Indicates that we are in tez testing mode. + + + + diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java index 1a84024..7afcd47 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java @@ -103,7 +103,7 @@ protected void setUp() { db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, src, true, true); db.createTable(src, cols, null, TextInputFormat.class, IgnoreKeyTextOutputFormat.class); - db.loadTable(hadoopDataFile[i], src, false, false, false, false); + db.loadTable(hadoopDataFile[i], src, false, false, false, false, false); i++; } diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java index 9807497..0c56132 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java @@ -147,10 +147,10 @@ public void testStatsAfterCompactionPartTbl() throws Exception { execSelectAndDumpData("select * from " + tblNameStg, driver, "Dumping data for " + tblNameStg + " after load:"); executeStatementOnDriver("FROM " + tblNameStg + - " INSERT OVERWRITE TABLE " + tblName + " PARTITION(bkt=0) " + + " INSERT INTO TABLE " + tblName + " PARTITION(bkt=0) " + "SELECT a, b where a < 2", driver); executeStatementOnDriver("FROM " + tblNameStg + - " INSERT OVERWRITE TABLE " + tblName + " PARTITION(bkt=1) " + + " INSERT INTO TABLE " + tblName + " PARTITION(bkt=1) " + "SELECT a, b where a >= 2", driver); execSelectAndDumpData("select * from " + tblName, driver, "Dumping data for " + tblName + " after load:"); diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index 0c4809f..6df9abb 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -63,6 +63,14 @@ minitez.query.files.shared=alter_merge_2_orc.q,\ cross_product_check_2.q,\ ctas.q,\ custom_input_output_format.q,\ + delete_all_non_partitioned.q,\ + delete_all_partitioned.q,\ + delete_orig_table.q,\ + delete_tmp_table.q,\ + delete_where_no_match.q,\ + delete_where_non_partitioned.q,\ + delete_where_partitioned.q,\ + delete_whole_partition.q,\ disable_merge_for_bucketing.q,\ dynpart_sort_opt_vectorization.q,\ dynpart_sort_optimization.q,\ @@ -76,6 +84,13 @@ minitez.query.files.shared=alter_merge_2_orc.q,\ insert1.q,\ insert_into1.q,\ insert_into2.q,\ + insert_orig_table.q,\ + insert_values_dynamic_partitioned.q,\ + insert_values_non_partitioned.q,\ + insert_values_orig_table.q\ + insert_values_partitioned.q,\ + insert_values_tmp_table.q,\ + insert_update_delete.q,\ join0.q,\ join1.q,\ leftsemijoin.q,\ @@ -122,6 +137,16 @@ minitez.query.files.shared=alter_merge_2_orc.q,\ union7.q,\ union8.q,\ union9.q,\ + update_after_multiple_inserts.q,\ + update_all_non_partitioned.q,\ + update_all_partitioned.q,\ + update_all_types.q,\ + update_orig_table.q,\ + update_tmp_table.q,\ + update_where_no_match.q,\ + update_where_non_partitioned.q,\ + update_where_partitioned.q,\ + update_two_cols.q,\ vector_cast_constant.q,\ vector_data_types.q,\ vector_decimal_aggregate.q,\ diff --git metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java index f1697bb..b71bb41 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java @@ -1081,7 +1081,8 @@ public int compare(LockInfo info1, LockInfo info2) { private static Map>> jumpTable; private void checkQFileTestHack() { - boolean hackOn = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_IN_TEST); + boolean hackOn = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_IN_TEST) || + HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_IN_TEZ_TEST); if (hackOn) { LOG.info("Hacking in canned values for transaction manager"); // Set up the transaction/locking db in the derby metastore diff --git ql/src/java/org/apache/hadoop/hive/ql/Context.java ql/src/java/org/apache/hadoop/hive/ql/Context.java index 7fcbe3c..0373273 100644 --- ql/src/java/org/apache/hadoop/hive/ql/Context.java +++ ql/src/java/org/apache/hadoop/hive/ql/Context.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.TaskRunner; import org.apache.hadoop.hive.ql.hooks.WriteEntity; +import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.lockmgr.HiveLock; import org.apache.hadoop.hive.ql.lockmgr.HiveLockManager; import org.apache.hadoop.hive.ql.lockmgr.HiveLockObj; @@ -98,6 +99,11 @@ // Transaction manager for this query protected HiveTxnManager hiveTxnManager; + // Used to track what type of acid operation (insert, update, or delete) we are doing. Useful + // since we want to change where bucket columns are accessed in some operators and + // optimizations when doing updates and deletes. + private AcidUtils.Operation acidOperation = AcidUtils.Operation.NOT_ACID; + private boolean needLockMgr; // Keep track of the mapping from load table desc to the output and the lock @@ -612,4 +618,12 @@ public int getTryCount() { public void setTryCount(int tryCount) { this.tryCount = tryCount; } + + public void setAcidOperation(AcidUtils.Operation op) { + acidOperation = op; + } + + public AcidUtils.Operation getAcidOperation() { + return acidOperation; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/Driver.java ql/src/java/org/apache/hadoop/hive/ql/Driver.java index f85a621..298f429 100644 --- ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -96,6 +96,7 @@ import org.apache.hadoop.hive.ql.parse.SemanticAnalyzerFactory; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.parse.VariableSubstitution; +import org.apache.hadoop.hive.ql.plan.FileSinkDesc; import org.apache.hadoop.hive.ql.plan.HiveOperation; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.TableDesc; @@ -137,6 +138,9 @@ private String SQLState; private Throwable downstreamError; + // A list of FileSinkOperators writing in an ACID compliant manner + private Set acidSinks; + // A limit on the number of threads that can be launched private int maxthreads; private int tryCount = Integer.MAX_VALUE; @@ -408,6 +412,9 @@ public int compile(String command, boolean resetTaskIds) { } else { sem.analyze(tree, ctx); } + // Record any ACID compliant FileSinkOperators we saw so we can add our transaction ID to + // them later. + acidSinks = sem.getAcidFileSinks(); LOG.info("Semantic Analysis Completed"); @@ -723,6 +730,11 @@ private static void doAuthorizationV2(SessionState ss, HiveOperation op, HashSet //do not authorize temporary uris continue; } + if (privObject instanceof ReadEntity && ((ReadEntity)privObject).isUpdateOrDelete()) { + // Skip this one, as we don't want to check select privileges for the table we're reading + // for an update or delete. + continue; + } //support for authorization on partitions needs to be added String dbname = null; @@ -859,7 +871,9 @@ public QueryPlan getPlan() { private int recordValidTxns() { try { ValidTxnList txns = SessionState.get().getTxnMgr().getValidTxns(); - conf.set(ValidTxnList.VALID_TXNS_KEY, txns.toString()); + String txnStr = txns.toString(); + conf.set(ValidTxnList.VALID_TXNS_KEY, txnStr); + LOG.debug("Encoding valid txns info " + txnStr); return 0; } catch (LockException e) { errorMessage = "FAILED: Error in determing valid transactions: " + e.getMessage(); @@ -877,13 +891,44 @@ private int recordValidTxns() { * pretty simple. If all the locks cannot be obtained, error out. Deadlock is avoided by making * sure that the locks are lexicographically sorted. **/ - private int acquireReadWriteLocks() { + private int acquireLocksAndOpenTxn() { PerfLogger perfLogger = PerfLogger.getPerfLogger(); perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.ACQUIRE_READ_WRITE_LOCKS); + SessionState ss = SessionState.get(); + HiveTxnManager txnMgr = ss.getTxnMgr(); try { - SessionState.get().getTxnMgr().acquireLocks(plan, ctx, userName); + // Don't use the userName member, as it may or may not have been set. Get the value from + // conf, which calls into getUGI to figure out who the process is running as. + String userFromUGI; + try { + userFromUGI = conf.getUser(); + } catch (IOException e) { + errorMessage = "FAILED: Error in determining user while acquiring locks: " + e.getMessage(); + SQLState = ErrorMsg.findSQLState(e.getMessage()); + downstreamError = e; + console.printError(errorMessage, + "\n" + org.apache.hadoop.util.StringUtils.stringifyException(e)); + return 10; + } + if (acidSinks != null && acidSinks.size() > 0) { + // We are writing to tables in an ACID compliant way, so we need to open a transaction + long txnId = ss.getCurrentTxn(); + if (txnId == SessionState.NO_CURRENT_TXN) { + txnId = txnMgr.openTxn(userFromUGI); + ss.setCurrentTxn(txnId); + } + // Set the transaction id in all of the acid file sinks + if (acidSinks != null) { + for (FileSinkDesc desc : acidSinks) { + desc.setTransactionId(txnId); + } + } + } + + txnMgr.acquireLocks(plan, ctx, userFromUGI); + return 0; } catch (LockException e) { errorMessage = "FAILED: Error in acquiring locks: " + e.getMessage(); @@ -901,13 +946,33 @@ private int acquireReadWriteLocks() { * @param hiveLocks * list of hive locks to be released Release all the locks specified. If some of the * locks have already been released, ignore them + * @param commit if there is an open transaction and if true, commit, + * if false rollback. If there is no open transaction this parameter is ignored. + * **/ - private void releaseLocks(List hiveLocks) throws LockException { + private void releaseLocksAndCommitOrRollback(List hiveLocks, boolean commit) + throws LockException { PerfLogger perfLogger = PerfLogger.getPerfLogger(); perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.RELEASE_LOCKS); - if (hiveLocks != null) { - SessionState.get().getTxnMgr().getLockManager().releaseLocks(hiveLocks); + SessionState ss = SessionState.get(); + HiveTxnManager txnMgr = ss.getTxnMgr(); + // If we've opened a transaction we need to commit or rollback rather than explicitly + // releasing the locks. + if (ss.getCurrentTxn() != SessionState.NO_CURRENT_TXN && ss.isAutoCommit()) { + try { + if (commit) { + txnMgr.commitTxn(); + } else { + txnMgr.rollbackTxn(); + } + } finally { + ss.setCurrentTxn(SessionState.NO_CURRENT_TXN); + } + } else { + if (hiveLocks != null) { + txnMgr.getLockManager().releaseLocks(hiveLocks); + } } ctx.setHiveLocks(null); @@ -994,7 +1059,7 @@ private int compileInternal(String command) { } if (ret != 0) { try { - releaseLocks(ctx.getHiveLocks()); + releaseLocksAndCommitOrRollback(ctx.getHiveLocks(), false); } catch (LockException e) { LOG.warn("Exception in releasing locks. " + org.apache.hadoop.util.StringUtils.stringifyException(e)); @@ -1097,10 +1162,10 @@ private CommandProcessorResponse runInternal(String command, boolean alreadyComp } if (requireLock) { - ret = acquireReadWriteLocks(); + ret = acquireLocksAndOpenTxn(); if (ret != 0) { try { - releaseLocks(ctx.getHiveLocks()); + releaseLocksAndCommitOrRollback(ctx.getHiveLocks(), false); } catch (LockException e) { // Not much to do here } @@ -1112,7 +1177,7 @@ private CommandProcessorResponse runInternal(String command, boolean alreadyComp if (ret != 0) { //if needRequireLock is false, the release here will do nothing because there is no lock try { - releaseLocks(ctx.getHiveLocks()); + releaseLocksAndCommitOrRollback(ctx.getHiveLocks(), false); } catch (LockException e) { // Nothing to do here } @@ -1121,7 +1186,7 @@ private CommandProcessorResponse runInternal(String command, boolean alreadyComp //if needRequireLock is false, the release here will do nothing because there is no lock try { - releaseLocks(ctx.getHiveLocks()); + releaseLocksAndCommitOrRollback(ctx.getHiveLocks(), true); } catch (LockException e) { errorMessage = "FAILED: Hive Internal Error: " + Utilities.getNameMessage(e); SQLState = ErrorMsg.findSQLState(e.getMessage()); @@ -1666,7 +1731,7 @@ public void destroy() { destroyed = true; if (ctx != null) { try { - releaseLocks(ctx.getHiveLocks()); + releaseLocksAndCommitOrRollback(ctx.getHiveLocks(), false); } catch (LockException e) { LOG.warn("Exception when releasing locking in destroy: " + e.getMessage()); diff --git ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index 4246d68..b3389cc 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -404,6 +404,19 @@ "time."), DISTINCT_NOT_SUPPORTED(10285, "Distinct keyword is not support in current context"), + UPDATEDELETE_PARSE_ERROR(10290, "Encountered parse error while parsing rewritten update or " + + "delete query"), + UPDATEDELETE_IO_ERROR(10291, "Encountered I/O error while parsing rewritten update or " + + "delete query"), + UPDATE_CANNOT_UPDATE_PART_VALUE(10292, "Updating values of partition columns is not supported"), + INSERT_CANNOT_CREATE_TEMP_FILE(10293, "Unable to create temp file for insert values "), + ACID_OP_ON_NONACID_TXNMGR(10294, "Attempt to do update or delete using transaction manager that" + + " does not support these operations."), + NO_INSERT_OVERWRITE_WITH_ACID(10295, "INSERT OVERWRITE not allowed on table with OutputFormat " + + "that implements AcidOutputFormat while transaction manager that supports ACID is in use"), + VALUES_TABLE_CONSTRUCTOR_NOT_SUPPORTED(10296, + "Values clause with table constructor not yet supported"), + //========================== 20000 range starts here ========================// SCRIPT_INIT_ERROR(20000, "Unable to initialize custom script."), SCRIPT_IO_ERROR(20001, "An error occurred while reading or writing to your custom script. " @@ -460,7 +473,10 @@ "to fail because of this, set hive.stats.atomic=false", true), STATS_SKIPPING_BY_ERROR(30017, "Skipping stats aggregation by error {0}", true), ORC_CORRUPTED_READ(30018, "Corruption in ORC data encountered. To skip reading corrupted " - + "data, set " + HiveConf.ConfVars.HIVE_ORC_SKIP_CORRUPT_DATA + " to true"); + + "data, set " + HiveConf.ConfVars.HIVE_ORC_SKIP_CORRUPT_DATA + " to true"), + + + ; private int errorCode; diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java index 7477199..66d6426 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hive.common.HiveStatsUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.ql.Context; @@ -35,6 +36,7 @@ import org.apache.hadoop.hive.ql.exec.mr.MapredLocalTask; import org.apache.hadoop.hive.ql.hooks.LineageInfo.DataContainer; import org.apache.hadoop.hive.ql.hooks.WriteEntity; +import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils; import org.apache.hadoop.hive.ql.io.merge.MergeTask; import org.apache.hadoop.hive.ql.lockmgr.HiveLock; @@ -274,7 +276,8 @@ public int execute(DriverContext driverContext) { dc = new DataContainer(table.getTTable()); db.loadTable(tbd.getSourcePath(), tbd.getTable() .getTableName(), tbd.getReplace(), tbd.getHoldDDLTime(), work.isSrcLocal(), - isSkewedStoredAsDirs(tbd)); + isSkewedStoredAsDirs(tbd), + work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID); if (work.getOutputs() != null) { work.getOutputs().add(new WriteEntity(table, (tbd.getReplace() ? WriteEntity.WriteType.INSERT_OVERWRITE : @@ -354,7 +357,8 @@ public int execute(DriverContext driverContext) { tbd.getReplace(), dpCtx.getNumDPCols(), tbd.getHoldDDLTime(), - isSkewedStoredAsDirs(tbd)); + isSkewedStoredAsDirs(tbd), + work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID); if (dp.size() == 0 && conf.getBoolVar(HiveConf.ConfVars.HIVE_ERROR_ON_EMPTY_PARTITION)) { throw new HiveException("This query creates no partitions." + @@ -389,7 +393,10 @@ public int execute(DriverContext driverContext) { // update columnar lineage for each partition dc = new DataContainer(table.getTTable(), partn.getTPartition()); - if (SessionState.get() != null) { + // Don't set lineage on delete as we don't have all the columns + if (SessionState.get() != null && + work.getLoadTableWork().getWriteType() != AcidUtils.Operation.DELETE && + work.getLoadTableWork().getWriteType() != AcidUtils.Operation.UPDATE) { SessionState.get().getLineageState().setLineage(tbd.getSourcePath(), dc, table.getCols()); } @@ -403,7 +410,8 @@ public int execute(DriverContext driverContext) { db.validatePartitionNameCharacters(partVals); db.loadPartition(tbd.getSourcePath(), tbd.getTable().getTableName(), tbd.getPartitionSpec(), tbd.getReplace(), tbd.getHoldDDLTime(), - tbd.getInheritTableSpecs(), isSkewedStoredAsDirs(tbd), work.isSrcLocal()); + tbd.getInheritTableSpecs(), isSkewedStoredAsDirs(tbd), work.isSrcLocal(), + work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID); Partition partn = db.getPartition(table, tbd.getPartitionSpec(), false); @@ -422,8 +430,24 @@ public int execute(DriverContext driverContext) { } } if (SessionState.get() != null && dc != null) { - SessionState.get().getLineageState().setLineage(tbd.getSourcePath(), dc, - table.getCols()); + // If we are doing an update or a delete the number of columns in the table will not + // match the number of columns in the file sink. For update there will be one too many + // (because of the ROW__ID), and in the case of the delete there will be just the + // ROW__ID, which we don't need to worry about from a lineage perspective. + List tableCols = null; + switch (work.getLoadTableWork().getWriteType()) { + case DELETE: + case UPDATE: + // Pass an empty list as no columns will be written to the file. + // TODO I should be able to make this work for update + tableCols = new ArrayList(); + break; + + default: + tableCols = table.getCols(); + break; + } + SessionState.get().getLineageState().setLineage(tbd.getSourcePath(), dc, tableCols); } releaseLocks(tbd); } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java index f018ca0..9bbc4ec 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java @@ -29,6 +29,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.io.HiveKey; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; @@ -43,8 +44,10 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.StandardUnionObjectInspector.StandardUnion; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; import org.apache.hadoop.io.BinaryComparable; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.IntWritable; @@ -131,10 +134,18 @@ // TODO: we only ever use one row of these at a time. Why do we need to cache multiple? protected transient Object[][] cachedKeys; + private StructField recIdField; // field to look for record identifier in + private StructField bucketField; // field to look for bucket in record identifier + private StructObjectInspector acidRowInspector; // row inspector used by acid options + private StructObjectInspector recIdInspector; // OI for the record identifier + private IntObjectInspector bucketInspector; // OI for the bucket field in the record id + @Override protected void initializeOp(Configuration hconf) throws HiveException { try { List keys = conf.getKeyCols(); + LOG.debug("keys size is " + keys.size()); + for (ExprNodeDesc k : keys) LOG.debug("Key exprNodeDesc " + k.getExprString()); keyEval = new ExprNodeEvaluator[keys.size()]; int i = 0; for (ExprNodeDesc e : keys) { @@ -259,6 +270,20 @@ public void processOp(Object row, int tag) throws HiveException { // TODO: this is fishy - we init object inspectors based on first tag. We // should either init for each tag, or if rowInspector doesn't really // matter, then we can create this in ctor and get rid of firstRow. + if (conf.getWriteType() == AcidUtils.Operation.UPDATE || + conf.getWriteType() == AcidUtils.Operation.DELETE) { + assert rowInspector instanceof StructObjectInspector : + "Exptected rowInspector to be instance of StructObjectInspector but it is a " + + rowInspector.getClass().getName(); + acidRowInspector = (StructObjectInspector)rowInspector; + // The record identifier is always in the first column + recIdField = acidRowInspector.getAllStructFieldRefs().get(0); + recIdInspector = (StructObjectInspector)recIdField.getFieldObjectInspector(); + // The bucket field is in the second position + bucketField = recIdInspector.getAllStructFieldRefs().get(1); + bucketInspector = (IntObjectInspector)bucketField.getFieldObjectInspector(); + } + LOG.info("keys are " + conf.getOutputKeyColumnNames() + " num distributions: " + conf.getNumDistributionKeys()); keyObjectInspector = initEvaluatorsAndReturnStruct(keyEval, distinctColIndices, @@ -283,6 +308,11 @@ public void processOp(Object row, int tag) throws HiveException { if (bucketEval != null) { buckNum = computeBucketNumber(row, conf.getNumBuckets()); cachedKeys[0][buckColIdxInKey] = new IntWritable(buckNum); + } else if (conf.getWriteType() == AcidUtils.Operation.UPDATE || + conf.getWriteType() == AcidUtils.Operation.DELETE) { + // In the non-partitioned case we still want to compute the bucket number for updates and + // deletes. + buckNum = computeBucketNumber(row, conf.getNumBuckets()); } HiveKey firstKey = toHiveKey(cachedKeys[0], tag, null); @@ -339,9 +369,20 @@ public void processOp(Object row, int tag) throws HiveException { private int computeBucketNumber(Object row, int numBuckets) throws HiveException { int buckNum = 0; - for (int i = 0; i < bucketEval.length; i++) { - Object o = bucketEval[i].evaluate(row); - buckNum = buckNum * 31 + ObjectInspectorUtils.hashCode(o, bucketObjectInspectors[i]); + + if (conf.getWriteType() == AcidUtils.Operation.UPDATE || + conf.getWriteType() == AcidUtils.Operation.DELETE) { + // We don't need to evalute the hash code. Instead read the bucket number directly from + // the row. I don't need to evaluate any expressions as I know I am reading the ROW__ID + // column directly. + Object recIdValue = acidRowInspector.getStructFieldData(row, recIdField); + buckNum = bucketInspector.get(recIdInspector.getStructFieldData(recIdValue, bucketField)); + LOG.debug("Acid choosing bucket number " + buckNum); + } else { + for (int i = 0; i < bucketEval.length; i++) { + Object o = bucketEval[i].evaluate(row); + buckNum = buckNum * 31 + ObjectInspectorUtils.hashCode(o, bucketObjectInspectors[i]); + } } if (buckNum < 0) { @@ -385,14 +426,19 @@ private int computeHashCode(Object row, int buckNum) throws HiveException { // Evaluate the HashCode int keyHashCode = 0; if (partitionEval.length == 0) { - // If no partition cols, just distribute the data uniformly to provide better - // load balance. If the requirement is to have a single reducer, we should set - // the number of reducers to 1. - // Use a constant seed to make the code deterministic. - if (random == null) { - random = new Random(12345); + // If no partition cols and not doing an update or delete, just distribute the data uniformly + // to provide better load balance. If the requirement is to have a single reducer, we should + // set the number of reducers to 1. Use a constant seed to make the code deterministic. + // For acid operations make sure to send all records with the same key to the same + // FileSinkOperator, as the RecordUpdater interface can't manage multiple writers for a file. + if (conf.getWriteType() == AcidUtils.Operation.NOT_ACID) { + if (random == null) { + random = new Random(12345); + } + keyHashCode = random.nextInt(); + } else { + keyHashCode = 1; } - keyHashCode = random.nextInt(); } else { for (int i = 0; i < partitionEval.length; i++) { Object o = partitionEval[i].evaluate(row); @@ -400,6 +446,7 @@ private int computeHashCode(Object row, int buckNum) throws HiveException { + ObjectInspectorUtils.hashCode(o, partitionObjectInspectors[i]); } } + LOG.debug("Going to return hash code " + (keyHashCode * 31 + buckNum)); return buckNum < 0 ? keyHashCode : keyHashCode * 31 + buckNum; } diff --git ql/src/java/org/apache/hadoop/hive/ql/hooks/ReadEntity.java ql/src/java/org/apache/hadoop/hive/ql/hooks/ReadEntity.java index e3bc3b1..8b6a923 100644 --- ql/src/java/org/apache/hadoop/hive/ql/hooks/ReadEntity.java +++ ql/src/java/org/apache/hadoop/hive/ql/hooks/ReadEntity.java @@ -48,6 +48,11 @@ // is marked as being read. Defaults to true as that is the most common case. private boolean needsLock = true; + // When true indicates that this object is being read as part of an update or delete. This is + // important because in that case we shouldn't acquire a lock for it or authorize the read. + // These will be handled by the output to the table instead. + private boolean isUpdateOrDelete = false; + // For views, the entities can be nested - by default, entities are at the top level private final Set parents = new HashSet(); @@ -166,4 +171,12 @@ public void noLockNeeded() { public List getAccessedColumns() { return accessedColumns; } + + public void setUpdateOrDelete(boolean isUpdateOrDelete) { + this.isUpdateOrDelete = isUpdateOrDelete; + } + + public boolean isUpdateOrDelete() { + return isUpdateOrDelete; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java index 7f1d71b..c5be822 100644 --- ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java +++ ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java @@ -148,6 +148,16 @@ public WriteType getWriteType() { } /** + * Only use this if you are very sure of what you are doing. This is used by the + * {@link org.apache.hadoop.hive.ql.parse.UpdateDeleteSemanticAnalyzer} to reset the types to + * update or delete after rewriting and reparsing the queries. + * @param type new operation type + */ + public void setWriteType(WriteType type) { + writeType = type; + } + + /** * Equals function. */ @Override diff --git ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java index b1c4441..80ef611 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java @@ -26,14 +26,12 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.hive.common.ValidTxnList; -import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.shims.HadoopShims; import org.apache.hadoop.hive.shims.ShimLoader; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.regex.Pattern; @@ -42,24 +40,40 @@ * are used by the compactor and cleaner and thus must be format agnostic. */ public class AcidUtils { - private AcidUtils() { - // NOT USED - } - private static final Log LOG = LogFactory.getLog(AcidUtils.class.getName()); - public static final String BASE_PREFIX = "base_"; public static final String DELTA_PREFIX = "delta_"; + public static final PathFilter deltaFileFilter = new PathFilter() { + @Override + public boolean accept(Path path) { + return path.getName().startsWith(DELTA_PREFIX); + } + }; public static final String BUCKET_PREFIX = "bucket_"; - + public static final PathFilter bucketFileFilter = new PathFilter() { + @Override + public boolean accept(Path path) { + return path.getName().startsWith(BUCKET_PREFIX); + } + }; public static final String BUCKET_DIGITS = "%05d"; public static final String DELTA_DIGITS = "%07d"; + public static final Pattern BUCKET_DIGIT_PATTERN = Pattern.compile("[0-9]{5}$"); + public static final Pattern LEGACY_BUCKET_DIGIT_PATTERN = Pattern.compile("^[0-9]{5}"); + public static final PathFilter originalBucketFilter = new PathFilter() { + @Override + public boolean accept(Path path) { + return ORIGINAL_PATTERN.matcher(path.getName()).matches(); + } + }; + + private AcidUtils() { + // NOT USED + } + private static final Log LOG = LogFactory.getLog(AcidUtils.class.getName()); private static final Pattern ORIGINAL_PATTERN = Pattern.compile("[0-9]+_[0-9]+"); - public static final Pattern BUCKET_DIGIT_PATTERN = Pattern.compile("[0-9]{5}$"); - public static final Pattern LEGACY_BUCKET_DIGIT_PATTERN = Pattern.compile("^[0-9]{5}"); - public static final PathFilter hiddenFileFilter = new PathFilter(){ public boolean accept(Path p){ String name = p.getName(); @@ -67,13 +81,6 @@ public boolean accept(Path p){ } }; - public static final PathFilter bucketFileFilter = new PathFilter() { - @Override - public boolean accept(Path path) { - return path.getName().startsWith(BUCKET_PREFIX); - } - }; - private static final HadoopShims SHIMS = ShimLoader.getHadoopShims(); /** @@ -149,7 +156,7 @@ static long parseBase(Path path) { .minimumTransactionId(0) .maximumTransactionId(0) .bucket(bucket); - } else if (filename.startsWith(AcidUtils.BUCKET_PREFIX)) { + } else if (filename.startsWith(BUCKET_PREFIX)) { int bucket = Integer.parseInt(filename.substring(filename.indexOf('_') + 1)); result @@ -372,7 +379,8 @@ public static Directory getAcidState(Path directory, } final Path base = bestBase == null ? null : bestBase.getPath(); - LOG.debug("base = " + base + " deltas = " + deltas.size()); + LOG.debug("in directory " + directory.toUri().toString() + " base = " + base + " deltas = " + + deltas.size()); return new Directory(){ diff --git ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java index 264052f..64c17dd 100644 --- ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java +++ ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java @@ -53,11 +53,12 @@ } @Override - public void openTxn(String user) throws LockException { + public long openTxn(String user) throws LockException { init(); try { txnId = client.openTxn(user); LOG.debug("Opened txn " + txnId); + return txnId; } catch (TException e) { throw new LockException(ErrorMsg.METASTORE_COMMUNICATION_FAILED.getMsg(), e); @@ -88,7 +89,11 @@ public void acquireLocks(QueryPlan plan, Context ctx, String username) throws Lo // For each source to read, get a shared lock for (ReadEntity input : plan.getInputs()) { - if (!input.needsLock()) continue; + if (!input.needsLock() || input.isUpdateOrDelete()) { + // We don't want to acquire readlocks during update or delete as we'll be acquiring write + // locks instead. + continue; + } LockComponentBuilder compBuilder = new LockComponentBuilder(); compBuilder.setShared(); @@ -297,6 +302,11 @@ public boolean useNewShowLocksFormat() { } @Override + public boolean supportsAcid() { + return true; + } + + @Override protected void destruct() { try { if (txnId > 0) rollbackTxn(); diff --git ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DummyTxnManager.java ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DummyTxnManager.java index 8354ad9..fdf6676 100644 --- ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DummyTxnManager.java +++ ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DummyTxnManager.java @@ -48,8 +48,9 @@ private HiveLockManager lockMgr; @Override - public void openTxn(String user) throws LockException { + public long openTxn(String user) throws LockException { // No-op + return 0L; } @Override @@ -208,6 +209,11 @@ public boolean useNewShowLocksFormat() { return false; } + @Override + public boolean supportsAcid() { + return false; + } + protected void destruct() { if (lockMgr != null) { diff --git ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java index 32d2f7a..2dd0c7d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java +++ ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java @@ -32,9 +32,10 @@ /** * Open a new transaction. * @param user Hive user who is opening this transaction. + * @return The new transaction id * @throws LockException if a transaction is already open. */ - void openTxn(String user) throws LockException; + long openTxn(String user) throws LockException; /** * Get the lock manager. This must be used rather than instantiating an @@ -120,4 +121,10 @@ * @return true if the new format should be used. */ boolean useNewShowLocksFormat(); + + /** + * Indicate whether this transaction manager supports ACID operations + * @return true if this transaction manager does ACID + */ + boolean supportsAcid(); } diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 2b1a345..f40f5f7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -96,6 +96,7 @@ import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.index.HiveIndexHandler; +import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.optimizer.listbucketingpruner.ListBucketingPrunerUtils; import org.apache.hadoop.hive.ql.plan.AddPartitionDesc; import org.apache.hadoop.hive.ql.plan.DropTableDesc; @@ -1227,7 +1228,7 @@ public Database getDatabaseCurrent() throws HiveException { public void loadPartition(Path loadPath, String tableName, Map partSpec, boolean replace, boolean holdDDLTime, boolean inheritTableSpecs, boolean isSkewedStoreAsSubdir, - boolean isSrcLocal) throws HiveException { + boolean isSrcLocal, boolean isAcid) throws HiveException { Table tbl = getTable(tableName); Path tblDataLocationPath = tbl.getDataLocation(); try { @@ -1275,7 +1276,7 @@ public void loadPartition(Path loadPath, String tableName, isSrcLocal); } else { FileSystem fs = tbl.getDataLocation().getFileSystem(conf); - Hive.copyFiles(conf, loadPath, newPartPath, fs, isSrcLocal); + Hive.copyFiles(conf, loadPath, newPartPath, fs, isSrcLocal, isAcid); } // recreate the partition if it existed before @@ -1407,7 +1408,7 @@ private void constructOneLBLocationMap(FileStatus fSta, */ public ArrayList> loadDynamicPartitions(Path loadPath, String tableName, Map partSpec, boolean replace, - int numDP, boolean holdDDLTime, boolean listBucketingEnabled) + int numDP, boolean holdDDLTime, boolean listBucketingEnabled, boolean isAcid) throws HiveException { Set validPartitions = new HashSet(); @@ -1463,7 +1464,7 @@ private void constructOneLBLocationMap(FileStatus fSta, // finally load the partition -- move the file to the final table address loadPartition(partPath, tableName, fullPartSpec, replace, holdDDLTime, true, - listBucketingEnabled, false); + listBucketingEnabled, false, isAcid); LOG.info("New loading path = " + partPath + " with partSpec " + fullPartSpec); } return fullPartSpecs; @@ -1489,14 +1490,16 @@ private void constructOneLBLocationMap(FileStatus fSta, * If the source directory is LOCAL * @param isSkewedStoreAsSubdir * if list bucketing enabled + * @param isAcid true if this is an ACID based write */ public void loadTable(Path loadPath, String tableName, boolean replace, - boolean holdDDLTime, boolean isSrcLocal, boolean isSkewedStoreAsSubdir) throws HiveException { + boolean holdDDLTime, boolean isSrcLocal, boolean isSkewedStoreAsSubdir, boolean isAcid) + throws HiveException { Table tbl = getTable(tableName); if (replace) { tbl.replaceFiles(loadPath, isSrcLocal); } else { - tbl.copyFiles(loadPath, isSrcLocal); + tbl.copyFiles(loadPath, isSrcLocal, isAcid); } try { @@ -2313,8 +2316,19 @@ public static boolean renameFile(HiveConf conf, Path srcf, Path destf, return success; } + /** + * Copy files. This handles building the mapping for buckets and such between the source and + * destination + * @param conf Configuration object + * @param srcf source directory, if bucketed should contain bucket files + * @param destf directory to move files into + * @param fs Filesystem + * @param isSrcLocal true if source is on local file system + * @param isAcid true if this is an ACID based write + * @throws HiveException + */ static protected void copyFiles(HiveConf conf, Path srcf, Path destf, - FileSystem fs, boolean isSrcLocal) throws HiveException { + FileSystem fs, boolean isSrcLocal, boolean isAcid) throws HiveException { boolean inheritPerms = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS); try { @@ -2342,23 +2356,105 @@ static protected void copyFiles(HiveConf conf, Path srcf, Path destf, return; // srcs = new FileStatus[0]; Why is this needed? } + + // If we're moving files around for an ACID write then the rules and paths are all different. + // You can blame this on Owen. + if (isAcid) { + moveAcidFiles(srcFs, srcs, destf); + } else { // check that source and target paths exist - List> result = checkPaths(conf, fs, srcs, srcFs, destf, false); - // move it, move it - try { - for (List sdpairs : result) { - for (Path[] sdpair : sdpairs) { - if (!renameFile(conf, sdpair[0], sdpair[1], fs, false, isSrcLocal)) { - throw new IOException("Cannot move " + sdpair[0] + " to " - + sdpair[1]); + List> result = checkPaths(conf, fs, srcs, srcFs, destf, false); + // move it, move it + try { + for (List sdpairs : result) { + for (Path[] sdpair : sdpairs) { + if (!renameFile(conf, sdpair[0], sdpair[1], fs, false, isSrcLocal)) { + throw new IOException("Cannot move " + sdpair[0] + " to " + + sdpair[1]); + } } } + } catch (IOException e) { + throw new HiveException("copyFiles: error while moving files!!!", e); } - } catch (IOException e) { - throw new HiveException("copyFiles: error while moving files!!!", e); } } + private static void moveAcidFiles(FileSystem fs, FileStatus[] stats, Path dst) + throws HiveException { + // The layout for ACID files is table|partname/base|delta/bucket + // We will always only be writing delta files. In the buckets created by FileSinkOperator + // it will look like bucket/delta/bucket. So we need to move that into the above structure. + // For the first mover there will be no delta directory, so we can move the whole directory. + // For everyone else we will need to just move the buckets under the existing delta + // directory. + + Set createdDeltaDirs = new HashSet(); + // Open the original path we've been given and find the list of original buckets + for (FileStatus stat : stats) { + Path srcPath = stat.getPath(); + + LOG.debug("Acid move Looking for original buckets in " + srcPath); + + FileStatus[] origBucketStats = null; + try { + origBucketStats = fs.listStatus(srcPath, AcidUtils.originalBucketFilter); + } catch (IOException e) { + String msg = "Unable to look for bucket files in src path " + srcPath.toUri().toString(); + LOG.error(msg); + throw new HiveException(msg, e); + } + LOG.debug("Acid move found " + origBucketStats.length + " original buckets"); + + for (FileStatus origBucketStat : origBucketStats) { + Path origBucketPath = origBucketStat.getPath(); + LOG.debug("Acid move looking for delta files in bucket " + origBucketPath); + + FileStatus[] deltaStats = null; + try { + deltaStats = fs.listStatus(origBucketPath, AcidUtils.deltaFileFilter); + } catch (IOException e) { + throw new HiveException("Unable to look for delta files in original bucket " + + origBucketPath.toUri().toString(), e); + } + LOG.debug("Acid move found " + deltaStats.length + " delta files"); + + for (FileStatus deltaStat : deltaStats) { + Path deltaPath = deltaStat.getPath(); + // Create the delta directory. Don't worry if it already exists, + // as that likely means another task got to it first. Then move each of the buckets. + // it would be more efficient to try to move the delta with it's buckets but that is + // harder to make race condition proof. + Path deltaDest = new Path(dst, deltaPath.getName()); + try { + if (!createdDeltaDirs.contains(deltaDest)) { + try { + fs.mkdirs(deltaDest); + createdDeltaDirs.add(deltaDest); + } catch (IOException swallowIt) { + // Don't worry about this, as it likely just means it's already been created. + LOG.info("Unable to create delta directory " + deltaDest + + ", assuming it already exists: " + swallowIt.getMessage()); + } + } + FileStatus[] bucketStats = fs.listStatus(deltaPath, AcidUtils.bucketFileFilter); + LOG.debug("Acid move found " + bucketStats.length + " bucket files"); + for (FileStatus bucketStat : bucketStats) { + Path bucketSrc = bucketStat.getPath(); + Path bucketDest = new Path(deltaDest, bucketSrc.getName()); + LOG.info("Moving bucket " + bucketSrc.toUri().toString() + " to " + + bucketDest.toUri().toString()); + fs.rename(bucketSrc, bucketDest); + } + } catch (IOException e) { + throw new HiveException("Error moving acid files", e); + } + } + } + } + } + + /** * Replaces files in the partition with new data set specified by srcf. Works * by renaming directory of srcf to the destination file. diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index 4acafba..c2e033c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -659,12 +659,14 @@ protected void replaceFiles(Path srcf, boolean isSrcLocal) * Files to be moved. Leaf directories or globbed file paths * @param isSrcLocal * If the source directory is LOCAL + * @param isAcid + * True if this is an ACID based insert, update, or delete */ - protected void copyFiles(Path srcf, boolean isSrcLocal) throws HiveException { + protected void copyFiles(Path srcf, boolean isSrcLocal, boolean isAcid) throws HiveException { FileSystem fs; try { fs = getDataLocation().getFileSystem(Hive.get().getConf()); - Hive.copyFiles(Hive.get().getConf(), srcf, getPath(), fs, isSrcLocal); + Hive.copyFiles(Hive.get().getConf(), srcf, getPath(), fs, isSrcLocal, isAcid); } catch (IOException e) { throw new HiveException("addFiles: filesystem error in check phase", e); } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java index 96a5d78..2f1497a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hive.ql.exec.SMBMapJoinOperator; import org.apache.hadoop.hive.ql.exec.SelectOperator; import org.apache.hadoop.hive.ql.exec.TableScanOperator; +import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker; import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; import org.apache.hadoop.hive.ql.lib.Dispatcher; @@ -371,6 +372,12 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, return null; } + // Don't do this optimization with updates or deletes + if (pGraphContext.getContext().getAcidOperation() == AcidUtils.Operation.UPDATE || + pGraphContext.getContext().getAcidOperation() == AcidUtils.Operation.DELETE){ + return null; + } + // Support for dynamic partitions can be added later if (fsOp.getConf().getDynPartCtx() != null) { return null; diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java index 5c711cf..cbb59ae 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java @@ -20,6 +20,7 @@ import java.io.Serializable; import java.util.ArrayList; +import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -42,6 +43,7 @@ import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; import org.apache.hadoop.hive.ql.exec.RowSchema; import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker; import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; import org.apache.hadoop.hive.ql.lib.Dispatcher; @@ -173,8 +175,22 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, destTable.getCols()); ObjectPair, List> sortOrderPositions = getSortPositionsOrder( destTable.getSortCols(), destTable.getCols()); - List sortPositions = sortOrderPositions.getFirst(); - List sortOrder = sortOrderPositions.getSecond(); + List sortPositions = null; + List sortOrder = null; + if (fsOp.getConf().getWriteType() == AcidUtils.Operation.UPDATE || + fsOp.getConf().getWriteType() == AcidUtils.Operation.DELETE) { + // When doing updates and deletes we always want to sort on the rowid because the ACID + // reader will expect this sort order when doing reads. So + // ignore whatever comes from the table and enforce this sort order instead. + sortPositions = Arrays.asList(0); + sortOrder = Arrays.asList(1); // 1 means asc, could really use enum here in the thrift if + } else { + sortPositions = sortOrderPositions.getFirst(); + sortOrder = sortOrderPositions.getSecond(); + } + LOG.debug("Got sort order"); + for (int i : sortPositions) LOG.debug("sort position " + i); + for (int i : sortOrder) LOG.debug("sort order " + i); List partitionPositions = getPartitionPositions(dpCtx, fsParent.getSchema()); List colInfos = parseCtx.getOpParseCtx().get(fsParent).getRowResolver() .getColumnInfos(); @@ -198,7 +214,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, colExprMap.put(ci.getInternalName(), newValueCols.get(newValueCols.size() - 1)); } ReduceSinkDesc rsConf = getReduceSinkDesc(partitionPositions, sortPositions, sortOrder, - newValueCols, bucketColumns, numBuckets, fsParent); + newValueCols, bucketColumns, numBuckets, fsParent, fsOp.getConf().getWriteType()); // Create ReduceSink operator ReduceSinkOperator rsOp = (ReduceSinkOperator) putOpInsertMap( @@ -319,7 +335,7 @@ private void removeRSInsertedByEnforceBucketing(FileSinkOperator fsOp) { public ReduceSinkDesc getReduceSinkDesc(List partitionPositions, List sortPositions, List sortOrder, ArrayList newValueCols, ArrayList bucketColumns, int numBuckets, - Operator parent) { + Operator parent, AcidUtils.Operation writeType) { // Order of KEY columns // 1) Partition columns @@ -409,7 +425,7 @@ public ReduceSinkDesc getReduceSinkDesc(List partitionPositions, // Number of reducers is set to default (-1) ReduceSinkDesc rsConf = new ReduceSinkDesc(newKeyCols, newKeyCols.size(), newValueCols, outputKeyCols, distinctColumnIndices, outValColNames, -1, newPartCols, -1, keyTable, - valueTable); + valueTable, writeType); rsConf.setBucketCols(bucketColumns); rsConf.setNumBuckets(numBuckets); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index 5195748..b2f66e0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -31,6 +31,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Set; import org.antlr.runtime.tree.CommonTree; import org.antlr.runtime.tree.Tree; @@ -60,6 +61,7 @@ import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.optimizer.listbucketingpruner.ListBucketingPrunerUtils; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.FileSinkDesc; import org.apache.hadoop.hive.ql.plan.ListBucketingCtx; import org.apache.hadoop.hive.ql.plan.PlanUtils; import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; @@ -89,6 +91,13 @@ protected HashMap idToTableNameMap; protected QueryProperties queryProperties; + /** + * A set of FileSinkOperators being written to in an ACID compliant way. We need to remember + * them here because when we build them we don't yet know the transaction id. We need to go + * back and set it once we actually start running the query. + */ + protected Set acidFileSinks = new HashSet(); + public static int HIVE_COLUMN_ORDER_ASC = 1; public static int HIVE_COLUMN_ORDER_DESC = 0; @@ -943,6 +952,10 @@ public QueryProperties getQueryProperties() { return queryProperties; } + public Set getAcidFileSinks() { + return acidFileSinks; + } + /** * Construct list bucketing context. * diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java index 911ac8a..86e4602 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java @@ -172,6 +172,15 @@ public boolean isInsertIntoTable(String dbName, String table) { return insertIntoTables.contains(fullName.toLowerCase()); } + /** + * Check if a table is in the list to be inserted into + * @param fullTableName table name in dbname.tablename format + * @return + */ + public boolean isInsertIntoTable(String fullTableName) { + return insertIntoTables.contains(fullTableName.toLowerCase()); + } + public HashMap getAggregationExprsForClause(String clause) { return destToAggregationExprs.get(clause); } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 496f6a6..392f7ce 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -37,6 +37,8 @@ import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; +import org.antlr.runtime.ClassicToken; +import org.antlr.runtime.Token; import org.antlr.runtime.tree.Tree; import org.antlr.runtime.tree.TreeWizard; import org.antlr.runtime.tree.TreeWizard.ContextVisitor; @@ -85,6 +87,8 @@ import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.hooks.WriteEntity; +import org.apache.hadoop.hive.ql.io.AcidOutputFormat; +import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.io.CombineHiveInputFormat; import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; import org.apache.hadoop.hive.ql.io.HiveOutputFormat; @@ -209,6 +213,8 @@ // Max characters when auto generating the column name with func name private static final int AUTOGEN_COLALIAS_PRFX_MAXLENGTH = 20; + private static final String VALUES_TMP_TABLE_NAME_PREFIX = "Values__Tmp__Table__"; + private HashMap opToPartPruner; private HashMap opToPartList; private HashMap> topOps; @@ -680,6 +686,140 @@ private String processTable(QB qb, ASTNode tabref) throws SemanticException { return alias; } + // Generate a temp table out of a value clause + private ASTNode genValuesTempTable(ASTNode originalFrom) throws SemanticException { + // Pick a name for the table + SessionState ss = SessionState.get(); + String tableName = VALUES_TMP_TABLE_NAME_PREFIX + ss.getNextValuesTempTableSuffix(); + + // Step 1, parse the values clause we were handed + List fromChildren = originalFrom.getChildren(); + // First child should be the virtual table ref + ASTNode virtualTableRef = (ASTNode)fromChildren.get(0); + assert virtualTableRef.getToken().getType() == HiveParser.TOK_VIRTUAL_TABREF : + "Expected first child of TOK_VIRTUAL_TABLE to be TOK_VIRTUAL_TABREF but was " + + virtualTableRef.getName(); + + List virtualTableRefChildren = virtualTableRef.getChildren(); + // First child of this should be the table name. If it's anonymous, + // then we don't have a table name. + ASTNode tabName = (ASTNode)virtualTableRefChildren.get(0); + if (tabName.getToken().getType() != HiveParser.TOK_ANONYMOUS) { + // TODO, if you want to make select ... from (values(...) as foo(...) work, + // you need to parse this list of columns names and build it into the table + throw new SemanticException(ErrorMsg.VALUES_TABLE_CONSTRUCTOR_NOT_SUPPORTED.getMsg()); + } + + // The second child of the TOK_VIRTUAL_TABLE should be TOK_VALUES_TABLE + ASTNode valuesTable = (ASTNode)fromChildren.get(1); + assert valuesTable.getToken().getType() == HiveParser.TOK_VALUES_TABLE : + "Expected second child of TOK_VIRTUAL_TABLE to be TOK_VALUE_TABLE but was " + + valuesTable.getName(); + // Each of the children of TOK_VALUES_TABLE will be a TOK_VALUE_ROW + List valuesTableChildren = valuesTable.getChildren(); + + // Now that we're going to start reading through the rows, open a file to write the rows too + // If we leave this method before creating the temporary table we need to be sure to clean up + // this file. + Path tablePath = null; + FileSystem fs = null; + try { + tablePath = Warehouse.getDnsPath(new Path(ss.getTempTableSpace(), tableName), conf); + fs = tablePath.getFileSystem(conf); + fs.mkdirs(tablePath); + Path dataFile = new Path(tablePath, "data_file"); + FSDataOutputStream out = fs.create(dataFile); + List fields = new ArrayList(); + + boolean firstRow = true; + for (Node n : valuesTableChildren) { + ASTNode valuesRow = (ASTNode) n; + assert valuesRow.getToken().getType() == HiveParser.TOK_VALUE_ROW : + "Expected child of TOK_VALUE_TABLE to be TOK_VALUE_ROW but was " + valuesRow.getName(); + // Each of the children of this should be a literal + List valuesRowChildren = valuesRow.getChildren(); + boolean isFirst = true; + int nextColNum = 1; + for (Node n1 : valuesRowChildren) { + ASTNode value = (ASTNode) n1; + if (firstRow) { + fields.add(new FieldSchema("tmp_values_col" + nextColNum++, "string", "")); + } + if (isFirst) isFirst = false; + else out.writeBytes("\u0001"); + out.writeBytes(unparseExprForValuesClause(value)); + } + out.writeBytes("\n"); + firstRow = false; + } + out.close(); + + // Step 2, create a temp table, using the created file as the data + StorageFormat format = new StorageFormat(conf); + format.processStorageFormat("TextFile"); + Table table = db.newTable(tableName); + table.setSerializationLib(format.getSerde()); + table.setFields(fields); + table.setDataLocation(tablePath); + table.getTTable().setTemporary(true); + table.setStoredAsSubDirectories(false); + table.setInputFormatClass(format.getInputFormat()); + table.setOutputFormatClass(format.getOutputFormat()); + db.createTable(table, false); + } catch (Exception e) { + String errMsg = ErrorMsg.INSERT_CANNOT_CREATE_TEMP_FILE.getMsg() + e.getMessage(); + LOG.error(errMsg); + // Try to delete the file + if (fs != null && tablePath != null) { + try { + fs.delete(tablePath, false); + } catch (IOException swallowIt) {} + } + throw new SemanticException(errMsg, e); + } + + // Step 3, return a new subtree with a from clause built around that temp table + // The form of the tree is TOK_TABREF->TOK_TABNAME->identifier(tablename) + Token t = new ClassicToken(HiveParser.TOK_TABREF); + ASTNode tabRef = new ASTNode(t); + t = new ClassicToken(HiveParser.TOK_TABNAME); + ASTNode tabNameNode = new ASTNode(t); + tabRef.addChild(tabNameNode); + t = new ClassicToken(HiveParser.Identifier, tableName); + ASTNode identifier = new ASTNode(t); + tabNameNode.addChild(identifier); + return tabRef; + } + + // Take an expression in the values clause and turn it back into a string. This is far from + // comprehensive. At the moment it only supports: + // * literals (all types) + // * unary negatives + // * true/false + private String unparseExprForValuesClause(ASTNode expr) throws SemanticException { + switch (expr.getToken().getType()) { + case HiveParser.Number: + return expr.getText(); + + case HiveParser.StringLiteral: + return PlanUtils.stripQuotes(expr.getText()); + + case HiveParser.KW_FALSE: + return "FALSE"; + + case HiveParser.KW_TRUE: + return "TRUE"; + + case HiveParser.MINUS: + return "-" + unparseExprForValuesClause((ASTNode)expr.getChildren().get(0)); + + default: + throw new SemanticException("Expression of type " + expr.getText() + + " not supported in insert/values"); + } + + } + private void assertCombineInputFormat(Tree numerator, String message) throws SemanticException { String inputFormat = conf.getVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez") ? HiveConf.getVar(conf, HiveConf.ConfVars.HIVETEZINPUTFORMAT): @@ -997,7 +1137,11 @@ public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1) if (frm.getToken().getType() == HiveParser.TOK_TABREF) { processTable(qb, frm); } else if (frm.getToken().getType() == HiveParser.TOK_VIRTUAL_TABLE) { - throw new RuntimeException("VALUES() clause is not fully supported yet..."); + // Create a temp table with the passed values in it then rewrite this portion of the + // tree to be from that table. + ASTNode newFrom = genValuesTempTable(frm); + ast.setChild(0, newFrom); + processTable(qb, newFrom); } else if (frm.getToken().getType() == HiveParser.TOK_SUBQUERY) { processSubQuery(qb, frm); } else if (frm.getToken().getType() == HiveParser.TOK_LATERAL_VIEW || @@ -1190,10 +1334,6 @@ public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1) case HiveParser.TOK_CTE: processCTE(qb, ast); break; - case HiveParser.TOK_DELETE_FROM: - throw new RuntimeException("DELETE is not (yet) implemented..."); - case HiveParser.TOK_UPDATE_TABLE: - throw new RuntimeException("UPDATE is not (yet) implemented..."); default: skipRecursion = false; break; @@ -1280,7 +1420,7 @@ public void getMetaData(QB qb, ReadEntity parentInput) throws SemanticException // Disallow INSERT INTO on bucketized tables if (qb.getParseInfo().isInsertIntoTable(tab.getDbName(), tab.getTableName()) && - tab.getNumBuckets() > 0) { + tab.getNumBuckets() > 0 && !isAcidTable(tab)) { throw new SemanticException(ErrorMsg.INSERT_INTO_BUCKETIZED_TABLE. getMsg("Table: " + tab_name)); } @@ -4226,7 +4366,7 @@ private ReduceSinkOperator genGroupByPlanReduceSinkOperator(QB qb, groupingSetsPresent ? keyLength + 1 : keyLength, reduceValues, distinctColIndices, outputKeyColumnNames, outputValueColumnNames, true, -1, numPartitionFields, - numReducers), + numReducers, AcidUtils.Operation.NOT_ACID), new RowSchema(reduceSinkOutputRowResolver.getColumnInfos()), inputOperatorInfo), reduceSinkOutputRowResolver); rsOp.setColumnExprMap(colExprMap); @@ -4429,7 +4569,7 @@ private ReduceSinkOperator genCommonGroupByPlanReduceSinkOperator(QB qb, List 0) && (conf.getBoolVar(HiveConf.ConfVars.HIVEENFORCEBUCKETING))) { enforceBucketing = true; - partnCols = getPartitionColsFromBucketCols(dest, qb, dest_tab, table_desc, input, true); - partnColsNoConvert = getPartitionColsFromBucketCols(dest, qb, dest_tab, table_desc, input, - false); + if (updating() || deleting()) { + partnCols = getPartitionColsFromBucketColsForUpdateDelete(input, true); + partnColsNoConvert = getPartitionColsFromBucketColsForUpdateDelete(input, false); + } else { + partnCols = getPartitionColsFromBucketCols(dest, qb, dest_tab, table_desc, input, true); + partnColsNoConvert = getPartitionColsFromBucketCols(dest, qb, dest_tab, table_desc, input, + false); + } } if ((dest_tab.getSortCols() != null) && @@ -5553,6 +5698,7 @@ private Operator genBucketingSortingDest(String dest, Operator input, QB qb, } int numBuckets = dest_tab.getNumBuckets(); if (numBuckets > maxReducers) { + LOG.debug("XXXXXX numBuckets is " + numBuckets + " and maxReducers is " + maxReducers); multiFileSpray = true; totalFiles = numBuckets; if (totalFiles % maxReducers == 0) { @@ -5718,7 +5864,11 @@ private Operator genFileSinkPlan(String dest, QB qb, Operator input) // Create the work for moving the table // NOTE: specify Dynamic partitions in dest_tab for WriteEntity if (!isNonNativeTable) { - ltd = new LoadTableDesc(queryTmpdir,table_desc, dpCtx); + AcidUtils.Operation acidOp = getAcidType(table_desc.getOutputFileFormatClass()); + if (acidOp != AcidUtils.Operation.NOT_ACID) { + checkIfAcidAndOverwriting(qb, table_desc); + } + ltd = new LoadTableDesc(queryTmpdir,table_desc, dpCtx, acidOp); ltd.setReplace(!qb.getParseInfo().isInsertIntoTable(dest_tab.getDbName(), dest_tab.getTableName())); ltd.setLbCtx(lbCtx); @@ -5821,7 +5971,11 @@ private Operator genFileSinkPlan(String dest, QB qb, Operator input) lbCtx = constructListBucketingCtx(dest_part.getSkewedColNames(), dest_part.getSkewedColValues(), dest_part.getSkewedColValueLocationMaps(), dest_part.isStoredAsSubDirectories(), conf); - ltd = new LoadTableDesc(queryTmpdir, table_desc, dest_part.getSpec()); + AcidUtils.Operation acidOp = getAcidType(table_desc.getOutputFileFormatClass()); + if (acidOp != AcidUtils.Operation.NOT_ACID) { + checkIfAcidAndOverwriting(qb, table_desc); + } + ltd = new LoadTableDesc(queryTmpdir, table_desc, dest_part.getSpec(), acidOp); ltd.setReplace(!qb.getParseInfo().isInsertIntoTable(dest_tab.getDbName(), dest_tab.getTableName())); ltd.setLbCtx(lbCtx); @@ -5973,18 +6127,25 @@ private Operator genFileSinkPlan(String dest, QB qb, Operator input) ArrayList vecCol = new ArrayList(); - try { - StructObjectInspector rowObjectInspector = (StructObjectInspector) table_desc - .getDeserializer().getObjectInspector(); - List fields = rowObjectInspector - .getAllStructFieldRefs(); - for (int i = 0; i < fields.size(); i++) { - vecCol.add(new ColumnInfo(fields.get(i).getFieldName(), TypeInfoUtils - .getTypeInfoFromObjectInspector(fields.get(i) - .getFieldObjectInspector()), "", false)); + if (updating() || deleting()) { + vecCol.add(new ColumnInfo(VirtualColumn.ROWID.getName(), + //TypeInfoUtils.getTypeInfoFromObjectInspector(VirtualColumn.ROWID.getObjectInspector()), + VirtualColumn.ROWID.getTypeInfo(), + "", true)); + } else { + try { + StructObjectInspector rowObjectInspector = (StructObjectInspector) table_desc + .getDeserializer().getObjectInspector(); + List fields = rowObjectInspector + .getAllStructFieldRefs(); + for (int i = 0; i < fields.size(); i++) { + vecCol.add(new ColumnInfo(fields.get(i).getFieldName(), TypeInfoUtils + .getTypeInfoFromObjectInspector(fields.get(i) + .getFieldObjectInspector()), "", false)); + } + } catch (Exception e) { + throw new SemanticException(e.getMessage(), e); } - } catch (Exception e) { - throw new SemanticException(e.getMessage(), e); } RowSchema fsRS = new RowSchema(vecCol); @@ -5997,6 +6158,10 @@ private Operator genFileSinkPlan(String dest, QB qb, Operator input) (dest_tab.getSortCols() != null && dest_tab.getSortCols().size() > 0 && conf.getBoolVar(HiveConf.ConfVars.HIVEENFORCESORTING)))); + // If this table is working with ACID semantics, turn off merging + boolean acidTable = isAcidTable(dest_tab); + canBeMerged &= !acidTable; + FileSinkDesc fileSinkDesc = new FileSinkDesc( queryTmpdir, table_desc, @@ -6009,6 +6174,15 @@ private Operator genFileSinkPlan(String dest, QB qb, Operator input) rsCtx.getPartnCols(), dpCtx); + // If this is an insert, update, or delete on an ACID table then mark that so the + // FileSinkOperator knows how to properly write to it. + if (acidTable) { + AcidUtils.Operation wt = updating() ? AcidUtils.Operation.UPDATE : + (deleting() ? AcidUtils.Operation.DELETE : AcidUtils.Operation.INSERT); + fileSinkDesc.setWriteType(wt); + acidFileSinks.add(fileSinkDesc); + } + /* Set List Bucketing context. */ if (lbCtx != null) { lbCtx.processRowSkewedIndex(fsRS); @@ -6059,6 +6233,17 @@ private Operator genFileSinkPlan(String dest, QB qb, Operator input) return output; } + // Check if we are overwriting any tables. If so, throw an exception as that is not allowed + // when using an Acid compliant txn manager and operating on an acid table. + private void checkIfAcidAndOverwriting(QB qb, TableDesc tableDesc) throws SemanticException { + String tableName = tableDesc.getTableName(); + if (!qb.getParseInfo().isInsertIntoTable(tableName)) { + LOG.debug("Couldn't find table " + tableName + " in insertIntoTable"); + throw new SemanticException(ErrorMsg.NO_INSERT_OVERWRITE_WITH_ACID.getMsg()); + } + + } + /** * Generate the conversion SelectOperator that converts the columns into the * types that are expected by the table_desc. @@ -6086,16 +6271,34 @@ Operator genConversionSelectOperator(String dest, QB qb, Operator input, outColumnCnt += dpCtx.getNumDPCols(); } - if (inColumnCnt != outColumnCnt) { - String reason = "Table " + dest + " has " + outColumnCnt - + " columns, but query has " + inColumnCnt + " columns."; - throw new SemanticException(ErrorMsg.TARGET_TABLE_COLUMN_MISMATCH.getMsg( - qb.getParseInfo().getDestForClause(dest), reason)); - } else if (dynPart && dpCtx != null) { - // create the mapping from input ExprNode to dest table DP column - dpCtx.mapInputToDP(rowFields.subList(tableFields.size(), rowFields.size())); + if (deleting()) { + // Figure out if we have partition columns in the list or not. If so, + // add them into the mapping. Partition columns will be located after the row id. + if (rowFields.size() > 1) { + // This means we have partition columns to deal with, so set up the mapping from the + // input to the partition columns. + dpCtx.mapInputToDP(rowFields.subList(1, rowFields.size())); + } + } else if (updating()) { + // In this case we expect the number of in fields to exceed the number of out fields by one + // (for the ROW__ID virtual column). If there are more columns than this, + // then the extras are for dynamic partitioning + if (dynPart && dpCtx != null) { + dpCtx.mapInputToDP(rowFields.subList(tableFields.size() + 1, rowFields.size())); + } + } else { + if (inColumnCnt != outColumnCnt) { + String reason = "Table " + dest + " has " + outColumnCnt + + " columns, but query has " + inColumnCnt + " columns."; + throw new SemanticException(ErrorMsg.TARGET_TABLE_COLUMN_MISMATCH.getMsg( + qb.getParseInfo().getDestForClause(dest), reason)); + } else if (dynPart && dpCtx != null) { + // create the mapping from input ExprNode to dest table DP column + dpCtx.mapInputToDP(rowFields.subList(tableFields.size(), rowFields.size())); + } } + // Check column types boolean converted = false; int columnNumber = tableFields.size(); @@ -6107,17 +6310,26 @@ Operator genConversionSelectOperator(String dest, QB qb, Operator input, MetadataTypedColumnsetSerDe.class); boolean isLazySimpleSerDe = table_desc.getDeserializerClass().equals( LazySimpleSerDe.class); - if (!isMetaDataSerDe) { + if (!isMetaDataSerDe && !deleting()) { + + // If we're updating, add the ROW__ID expression, then make the following column accesses + // offset by 1 so that we don't try to convert the ROW__ID + if (updating()) { + expressions.add(new ExprNodeColumnDesc(rowFields.get(0).getType(), + rowFields.get(0).getInternalName(), "", true)); + } // here only deals with non-partition columns. We deal with partition columns next for (int i = 0; i < columnNumber; i++) { + int rowFieldsOffset = updating() ? i + 1 : i; ObjectInspector tableFieldOI = tableFields.get(i) .getFieldObjectInspector(); TypeInfo tableFieldTypeInfo = TypeInfoUtils .getTypeInfoFromObjectInspector(tableFieldOI); - TypeInfo rowFieldTypeInfo = rowFields.get(i).getType(); + TypeInfo rowFieldTypeInfo = rowFields.get(rowFieldsOffset).getType(); ExprNodeDesc column = new ExprNodeColumnDesc(rowFieldTypeInfo, - rowFields.get(i).getInternalName(), "", false, rowFields.get(i).isSkewedCol()); + rowFields.get(rowFieldsOffset).getInternalName(), "", false, + rowFields.get(rowFieldsOffset).isSkewedCol()); // LazySimpleSerDe can convert any types to String type using // JSON-format. if (!tableFieldTypeInfo.equals(rowFieldTypeInfo) @@ -6147,7 +6359,7 @@ Operator genConversionSelectOperator(String dest, QB qb, Operator input, // deal with dynamic partition columns: convert ExprNodeDesc type to String?? if (dynPart && dpCtx != null && dpCtx.getNumDPCols() > 0) { // DP columns starts with tableFields.size() - for (int i = tableFields.size(); i < rowFields.size(); ++i) { + for (int i = tableFields.size() + (updating() ? 1 : 0); i < rowFields.size(); ++i) { TypeInfo rowFieldTypeInfo = rowFields.get(i).getType(); ExprNodeDesc column = new ExprNodeColumnDesc( rowFieldTypeInfo, rowFields.get(i).getInternalName(), "", false); @@ -6341,6 +6553,27 @@ private Operator genLimitMapRedPlan(String dest, QB qb, Operator input, return genConvertCol(dest, qb, tab, table_desc, input, posns, convert); } + // We have to set up the bucketing columns differently for update and deletes, + // as it is always using the ROW__ID column. + private ArrayList getPartitionColsFromBucketColsForUpdateDelete( + Operator input, boolean convert) throws SemanticException { + //return genConvertCol(dest, qb, tab, table_desc, input, Arrays.asList(0), convert); + // In the case of update and delete the bucketing column is always the first column, + // and it isn't in the table info. So rather than asking the table for it, + // we'll construct it ourself and send it back. This is based on the work done in + // genConvertCol below. + ColumnInfo rowField = opParseCtx.get(input).getRowResolver().getColumnInfos().get(0); + TypeInfo rowFieldTypeInfo = rowField.getType(); + ExprNodeDesc column = new ExprNodeColumnDesc(rowFieldTypeInfo, rowField.getInternalName(), + rowField.getTabAlias(), true); + if (convert) { + column = ParseUtils.createConversionCast(column, TypeInfoFactory.intTypeInfo); + } + ArrayList rlist = new ArrayList(1); + rlist.add(column); + return rlist; + } + private ArrayList genConvertCol(String dest, QB qb, Table tab, TableDesc table_desc, Operator input, List posns, boolean convert) throws SemanticException { @@ -6463,9 +6696,11 @@ private Operator genReduceSinkPlanForSortingBucketing(Table tab, Operator input, order.append(sortOrder == BaseSemanticAnalyzer.HIVE_COLUMN_ORDER_ASC ? '+' : '-'); } + AcidUtils.Operation acidOp = (isAcidTable(tab) ? getAcidType() : AcidUtils.Operation.NOT_ACID); + Operator interim = putOpInsertMap(OperatorFactory.getAndMakeChild(PlanUtils .getReduceSinkDesc(sortCols, valueCols, outputColumns, false, -1, - partitionCols, order.toString(), numReducers), + partitionCols, order.toString(), numReducers, acidOp), new RowSchema(inputRR.getColumnInfos()), input), inputRR); interim.setColumnExprMap(colExprMap); reduceSinkOperatorsAddedByEnforceBucketingSorting.add((ReduceSinkOperator) interim); @@ -6621,8 +6856,9 @@ private Operator genReduceSinkPlan(String dest, QB qb, Operator input, dummy.setParentOperators(null); + // TODO Not 100% sure NOT_ACID is always right here. ReduceSinkDesc rsdesc = PlanUtils.getReduceSinkDesc(sortCols, valueCols, outputColumns, - false, -1, partitionCols, order.toString(), numReducers); + false, -1, partitionCols, order.toString(), numReducers, AcidUtils.Operation.NOT_ACID); Operator interim = putOpInsertMap(OperatorFactory.getAndMakeChild(rsdesc, new RowSchema(rsRR.getColumnInfos()), input), rsRR); @@ -6887,7 +7123,7 @@ private Operator genJoinReduceSinkChild(QB qb, ExprNodeDesc[] joinKeys, ReduceSinkDesc rsDesc = PlanUtils.getReduceSinkDesc(reduceKeys, reduceValues, outputColumns, false, tag, - reduceKeys.size(), numReds); + reduceKeys.size(), numReds, AcidUtils.Operation.NOT_ACID); ReduceSinkOperator rsOp = (ReduceSinkOperator) putOpInsertMap( OperatorFactory.getAndMakeChild(rsDesc, new RowSchema(outputRR @@ -8075,7 +8311,8 @@ private Operator createCommonReduceSink(QB qb, Operator input) ReduceSinkOperator rsOp = (ReduceSinkOperator) putOpInsertMap( OperatorFactory.getAndMakeChild(PlanUtils.getReduceSinkDesc(reduceKeys, - reduceValues, outputColumnNames, true, -1, reduceKeys.size(), -1), + reduceValues, outputColumnNames, true, -1, reduceKeys.size(), -1, + AcidUtils.Operation.NOT_ACID), new RowSchema(reduceSinkOutputRowResolver.getColumnInfos()), input), reduceSinkOutputRowResolver); @@ -11375,7 +11612,7 @@ private Operator genPTFPlanForComponentQuery(PTFInvocationSpec ptfQSpec, Operato input = putOpInsertMap(OperatorFactory.getAndMakeChild(PlanUtils .getReduceSinkDesc(orderCols, valueCols, outputColumnNames, false, - -1, partCols, orderString.toString(), -1), + -1, partCols, orderString.toString(), -1, AcidUtils.Operation.NOT_ACID), new RowSchema(rsOpRR.getColumnInfos()), input), rsOpRR); input.setColumnExprMap(colExprMap); } @@ -11500,7 +11737,7 @@ private Operator genReduceSinkPlanForWindowing(WindowingSpec spec, input = putOpInsertMap(OperatorFactory.getAndMakeChild(PlanUtils .getReduceSinkDesc(orderCols, valueCols, outputColumnNames, false, - -1, partCols, orderString.toString(), -1), + -1, partCols, orderString.toString(), -1, AcidUtils.Operation.NOT_ACID), new RowSchema(rsNewRR.getColumnInfos()), input), rsNewRR); input.setColumnExprMap(colExprMap); @@ -11657,4 +11894,49 @@ private void addAlternateGByKeyMappings(ASTNode gByExpr, ColumnInfo colInfo, else return (ltd.getReplace() ? WriteEntity.WriteType.INSERT_OVERWRITE : WriteEntity.WriteType.INSERT); } + + // Even if the table is of Acid type, if we aren't working with an Acid compliant TxnManager + // then return false. + private boolean isAcidTable(Table tab) { + if (tab == null || tab.getOutputFormatClass() == null) return false; + if (!SessionState.get().getTxnMgr().supportsAcid()) return false; + return isAcidOutputFormat(tab.getOutputFormatClass()); + } + + private boolean isAcidOutputFormat(Class of) { + Class[] interfaces = of.getInterfaces(); + for (Class iface : interfaces) { + if (iface.equals(AcidOutputFormat.class)) { + return true; + } + } + return false; + } + + // Note that this method assumes you have already decided this is an Acid table. It cannot + // figure out if a table is Acid or not. + private AcidUtils.Operation getAcidType() { + return deleting() ? AcidUtils.Operation.DELETE : + (updating() ? AcidUtils.Operation.UPDATE : + AcidUtils.Operation.INSERT); + } + + private AcidUtils.Operation getAcidType(Class of) { + if (SessionState.get() == null || !SessionState.get().getTxnMgr().supportsAcid()) { + return AcidUtils.Operation.NOT_ACID; + } else if (isAcidOutputFormat(of)) { + return getAcidType(); + } else { + return AcidUtils.Operation.NOT_ACID; + } + } + + protected boolean updating() { + return false; + } + + protected boolean deleting() { + return false; + } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java index 3e3926e..6962ee9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java @@ -268,6 +268,11 @@ public static BaseSemanticAnalyzer get(HiveConf conf, ASTNode tree) case HiveParser.TOK_CREATEMACRO: case HiveParser.TOK_DROPMACRO: return new MacroSemanticAnalyzer(conf); + + case HiveParser.TOK_UPDATE_TABLE: + case HiveParser.TOK_DELETE_FROM: + return new UpdateDeleteSemanticAnalyzer(conf); + default: return new SemanticAnalyzer(conf); } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java index ad91b0f..7723430 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java @@ -80,7 +80,7 @@ public boolean fillStorageFormat(ASTNode child) throws SemanticException { return true; } - private void processStorageFormat(String name) throws SemanticException { + protected void processStorageFormat(String name) throws SemanticException { if (name.isEmpty()) { throw new SemanticException("File format in STORED AS clause cannot be empty"); } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java new file mode 100644 index 0000000..3aaa09c --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java @@ -0,0 +1,345 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.parse; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.ql.Context; +import org.apache.hadoop.hive.ql.ErrorMsg; +import org.apache.hadoop.hive.ql.hooks.Entity; +import org.apache.hadoop.hive.ql.hooks.ReadEntity; +import org.apache.hadoop.hive.ql.hooks.WriteEntity; +import org.apache.hadoop.hive.ql.io.AcidUtils; +import org.apache.hadoop.hive.ql.lib.Node; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.session.SessionState; + + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + + +/** + * A subclass of the {@link org.apache.hadoop.hive.ql.parse.SemanticAnalyzer} that just handles + * update and delete statements. It works by rewriting the updates and deletes into insert + * statements (since they are actually inserts) and then doing some patch up to make them work as + * updates and deletes instead. + */ +public class UpdateDeleteSemanticAnalyzer extends SemanticAnalyzer { + + boolean useSuper = false; + + public UpdateDeleteSemanticAnalyzer(HiveConf conf) throws SemanticException { + super(conf); + } + + @Override + public void analyzeInternal(ASTNode tree) throws SemanticException { + if (useSuper) { + super.analyzeInternal(tree); + } else { + + if (!SessionState.get().getTxnMgr().supportsAcid()) { + throw new SemanticException(ErrorMsg.ACID_OP_ON_NONACID_TXNMGR.getMsg()); + } + switch (tree.getToken().getType()) { + case HiveParser.TOK_DELETE_FROM: + analyzeDelete(tree); + return; + + case HiveParser.TOK_UPDATE_TABLE: + analyzeUpdate(tree); + return; + + default: + throw new RuntimeException("Asked to parse token " + tree.getName() + " in " + + "UpdateDeleteSemanticAnalyzer"); + } + } + } + + @Override + protected boolean updating() { + return ctx.getAcidOperation() == AcidUtils.Operation.UPDATE; + } + + @Override + protected boolean deleting() { + return ctx.getAcidOperation() == AcidUtils.Operation.DELETE; + } + + private void analyzeUpdate(ASTNode tree) throws SemanticException { + ctx.setAcidOperation(AcidUtils.Operation.UPDATE); + reparseAndSuperAnalyze(tree); + } + + private void analyzeDelete(ASTNode tree) throws SemanticException { + ctx.setAcidOperation(AcidUtils.Operation.DELETE); + reparseAndSuperAnalyze(tree); + } + + private void reparseAndSuperAnalyze(ASTNode tree) throws SemanticException { + List children = tree.getChildren(); + // The first child should be the table we are deleting from + ASTNode tabName = (ASTNode)children.get(0); + assert tabName.getToken().getType() == HiveParser.TOK_TABNAME : + "Expected tablename as first child of " + operation() + " but found " + tabName.getName(); + String[] tableName = getQualifiedTableName(tabName); + + // Rewrite the delete or update into an insert. Crazy, but it works as deletes and update + // actually are inserts into the delta file in Hive. A delete + // DELETE FROM _tablename_ [WHERE ...] + // will be rewritten as + // INSERT INTO TABLE _tablename_ [PARTITION (_partcols_)] SELECT ROW__ID[, + // _partcols_] from _tablename_ SORT BY ROW__ID + // An update + // UPDATE _tablename_ SET x = _expr_ [WHERE...] + // will be rewritten as + // INSERT INTO TABLE _tablename_ [PARTITION (_partcols_)] SELECT _all_, + // _partcols_from _tablename_ SORT BY ROW__ID + // where _all_ is all the non-partition columns. The expressions from the set clause will be + // re-attached later. + // The where clause will also be re-attached later. + // The sort by clause is put in there so that records come out in the right order to enable + // merge on read. + + StringBuilder rewrittenQueryStr = new StringBuilder(); + Table mTable; + try { + mTable = db.getTable(tableName[0], tableName[1]); + } catch (HiveException e) { + throw new SemanticException(ErrorMsg.UPDATEDELETE_PARSE_ERROR.getMsg(), e); + } + List partCols = mTable.getPartCols(); + + rewrittenQueryStr.append("insert into table "); + rewrittenQueryStr.append(getDotName(tableName)); + + // If the table is partitioned we have to put the partition() clause in + if (partCols != null && partCols.size() > 0) { + rewrittenQueryStr.append(" partition ("); + boolean first = true; + for (FieldSchema fschema : partCols) { + if (first) first = false; + else rewrittenQueryStr.append(", "); + rewrittenQueryStr.append(fschema.getName()); + } + rewrittenQueryStr.append(")"); + } + + rewrittenQueryStr.append(" select ROW__ID"); + Map setColExprs = null; + if (updating()) { + // An update needs to select all of the columns, as we rewrite the entire row. Also, + // we need to figure out which columns we are going to replace. We won't write the set + // expressions in the rewritten query. We'll patch that up later. + // The set list from update should be the second child (index 1) + assert children.size() >= 2 : "Expected update token to have at least two children"; + ASTNode setClause = (ASTNode)children.get(1); + assert setClause.getToken().getType() == HiveParser.TOK_SET_COLUMNS_CLAUSE : + "Expected second child of update token to be set token"; + + // Get the children of the set clause, each of which should be a column assignment + List assignments = setClause.getChildren(); + Map setCols = new HashMap(assignments.size()); + setColExprs = new HashMap(assignments.size()); + for (Node a : assignments) { + ASTNode assignment = (ASTNode)a; + assert assignment.getToken().getType() == HiveParser.EQUAL : + "Expected set assignments to use equals operator but found " + assignment.getName(); + ASTNode tableOrColTok = (ASTNode)assignment.getChildren().get(0); + assert tableOrColTok.getToken().getType() == HiveParser.TOK_TABLE_OR_COL : + "Expected left side of assignment to be table or column"; + ASTNode colName = (ASTNode)tableOrColTok.getChildren().get(0); + assert colName.getToken().getType() == HiveParser.Identifier : + "Expected column name"; + + String columnName = colName.getText(); + + // Make sure this isn't one of the partitioning columns, that's not supported. + if (partCols != null) { + for (FieldSchema fschema : partCols) { + if (fschema.getName().equalsIgnoreCase(columnName)) { + throw new SemanticException(ErrorMsg.UPDATE_CANNOT_UPDATE_PART_VALUE.getMsg()); + } + } + } + + // This means that in UPDATE T SET x = _something_ + // _something_ can be whatever is supported in SELECT _something_ + setCols.put(columnName, (ASTNode)assignment.getChildren().get(1)); + } + + List nonPartCols = mTable.getCols(); + for (int i = 0; i < nonPartCols.size(); i++) { + rewrittenQueryStr.append(','); + String name = nonPartCols.get(i).getName(); + ASTNode setCol = setCols.get(name); + rewrittenQueryStr.append(name); + if (setCol != null) { + // This is one of the columns we're setting, record it's position so we can come back + // later and patch it up. + // Add one to the index because the select has the ROW__ID as the first column. + setColExprs.put(i + 1, setCol); + } + } + } + + // If the table is partitioned, we need to select the partition columns as well. + if (partCols != null) { + for (FieldSchema fschema : partCols) { + rewrittenQueryStr.append(", "); + rewrittenQueryStr.append(fschema.getName()); + } + } + rewrittenQueryStr.append(" from "); + rewrittenQueryStr.append(getDotName(tableName)); + + ASTNode where = null; + int whereIndex = deleting() ? 1 : 2; + if (children.size() > whereIndex) { + where = (ASTNode)children.get(whereIndex); + assert where.getToken().getType() == HiveParser.TOK_WHERE : + "Expected where clause, but found " + where.getName(); + } + + // Add a sort by clause so that the row ids come out in the correct order + rewrittenQueryStr.append(" sort by ROW__ID desc "); + + // Parse the rewritten query string + Context rewrittenCtx; + try { + // Set dynamic partitioning to nonstrict so that queries do not need any partition + // references. + HiveConf.setVar(conf, HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict"); + rewrittenCtx = new Context(conf); + } catch (IOException e) { + throw new SemanticException(ErrorMsg.UPDATEDELETE_IO_ERROR.getMsg()); + } + rewrittenCtx.setCmd(rewrittenQueryStr.toString()); + rewrittenCtx.setAcidOperation(ctx.getAcidOperation()); + + ParseDriver pd = new ParseDriver(); + ASTNode rewrittenTree; + try { + LOG.info("Going to reparse " + operation() + " as <" + rewrittenQueryStr.toString() + ">"); + rewrittenTree = pd.parse(rewrittenQueryStr.toString(), rewrittenCtx); + rewrittenTree = ParseUtils.findRootNonNullToken(rewrittenTree); + + } catch (ParseException e) { + throw new SemanticException(ErrorMsg.UPDATEDELETE_PARSE_ERROR.getMsg(), e); + } + + ASTNode rewrittenInsert = (ASTNode)rewrittenTree.getChildren().get(1); + assert rewrittenInsert.getToken().getType() == HiveParser.TOK_INSERT : + "Expected TOK_INSERT as second child of TOK_QUERY but found " + rewrittenInsert.getName(); + + if (where != null) { + // The structure of the AST for the rewritten insert statement is: + // TOK_QUERY -> TOK_FROM + // \-> TOK_INSERT -> TOK_INSERT_INTO + // \-> TOK_SELECT + // \-> TOK_SORTBY + // The following adds the TOK_WHERE and its subtree from the original query as a child of + // TOK_INSERT, which is where it would have landed if it had been there originally in the + // string. We do it this way because it's easy then turning the original AST back into a + // string and reparsing it. We have to move the SORT_BY over one, + // so grab it and then push it to the second slot, and put the where in the first slot + ASTNode sortBy = (ASTNode)rewrittenInsert.getChildren().get(2); + assert sortBy.getToken().getType() == HiveParser.TOK_SORTBY : + "Expected TOK_SORTBY to be first child of TOK_SELECT, but found " + sortBy.getName(); + rewrittenInsert.addChild(sortBy); + rewrittenInsert.setChild(2, where); + } + + // Patch up the projection list for updates, putting back the original set expressions. + if (updating() && setColExprs != null) { + // Walk through the projection list and replace the column names with the + // expressions from the original update. Under the TOK_SELECT (see above) the structure + // looks like: + // TOK_SELECT -> TOK_SELEXPR -> expr + // \-> TOK_SELEXPR -> expr ... + ASTNode rewrittenSelect = (ASTNode)rewrittenInsert.getChildren().get(1); + assert rewrittenSelect.getToken().getType() == HiveParser.TOK_SELECT : + "Expected TOK_SELECT as second child of TOK_INSERT but found " + + rewrittenSelect.getName(); + for (Map.Entry entry : setColExprs.entrySet()) { + ASTNode selExpr = (ASTNode)rewrittenSelect.getChildren().get(entry.getKey()); + assert selExpr.getToken().getType() == HiveParser.TOK_SELEXPR : + "Expected child of TOK_SELECT to be TOK_SELEXPR but was " + selExpr.getName(); + // Now, change it's child + selExpr.setChild(0, entry.getValue()); + } + } + + try { + useSuper = true; + super.analyze(rewrittenTree, rewrittenCtx); + } finally { + useSuper = false; + } + + // Walk through all our inputs and set them to note that this read is part of an update or a + // delete. + for (ReadEntity input : inputs) { + input.setUpdateOrDelete(true); + } + + if (inputIsPartitioned(inputs)) { + // In order to avoid locking the entire write table we need to replace the single WriteEntity + // with a WriteEntity for each partition + outputs.clear(); + for (ReadEntity input : inputs) { + if (input.getTyp() == Entity.Type.PARTITION) { + WriteEntity.WriteType writeType = deleting() ? WriteEntity.WriteType.DELETE : + WriteEntity.WriteType.UPDATE; + outputs.add(new WriteEntity(input.getPartition(), writeType)); + } + } + } else { + // We still need to patch up the WriteEntities as they will have an insert type. Change + // them to the appropriate type for our operation. + for (WriteEntity output : outputs) { + output.setWriteType(deleting() ? WriteEntity.WriteType.DELETE : + WriteEntity.WriteType.UPDATE); + } + } + } + + private String operation() { + if (updating()) return "update"; + else if (deleting()) return "delete"; + else throw new IllegalStateException("UpdateDeleteSemanticAnalyzer neither updating nor " + + "deleting, operation not known."); + } + + private boolean inputIsPartitioned(Set inputs) { + // We cannot simply look at the first entry, as in the case where the input is partitioned + // there will be a table entry as well. So look for at least one partition entry. + for (ReadEntity re : inputs) { + if (re.getTyp() == Entity.Type.PARTITION) { + return true; + } + } + return false; + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java index 2dbf1c8..f514857 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java @@ -23,6 +23,7 @@ import java.util.Map; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.io.AcidUtils; /** * LoadTableDesc. @@ -37,6 +38,9 @@ private boolean holdDDLTime; private boolean inheritTableSpecs = true; //For partitions, flag controlling whether the current //table specs are to be used + // Need to remember whether this is an acid compliant operation, and if so whether it is an + // insert, update, or delete. + private AcidUtils.Operation writeType; // TODO: the below seems like they should just be combined into partitionDesc private org.apache.hadoop.hive.ql.plan.TableDesc table; @@ -48,36 +52,69 @@ public LoadTableDesc() { public LoadTableDesc(final Path sourcePath, final org.apache.hadoop.hive.ql.plan.TableDesc table, - final Map partitionSpec, final boolean replace) { + final Map partitionSpec, + final boolean replace, + final AcidUtils.Operation writeType) { super(sourcePath); - init(table, partitionSpec, replace); + init(table, partitionSpec, replace, writeType); + } + + /** + * For use with non-ACID compliant operations, such as LOAD + * @param sourcePath + * @param table + * @param partitionSpec + * @param replace + */ + public LoadTableDesc(final Path sourcePath, + final TableDesc table, + final Map partitionSpec, + final boolean replace) { + this(sourcePath, table, partitionSpec, replace, AcidUtils.Operation.NOT_ACID); } public LoadTableDesc(final Path sourcePath, final org.apache.hadoop.hive.ql.plan.TableDesc table, - final Map partitionSpec) { - this(sourcePath, table, partitionSpec, true); + final Map partitionSpec, + final AcidUtils.Operation writeType) { + this(sourcePath, table, partitionSpec, true, writeType); + } + + /** + * For DDL operations that are not ACID compliant. + * @param sourcePath + * @param table + * @param partitionSpec + */ + public LoadTableDesc(final Path sourcePath, + final org.apache.hadoop.hive.ql.plan.TableDesc table, + final Map partitionSpec) { + this(sourcePath, table, partitionSpec, true, AcidUtils.Operation.NOT_ACID); } public LoadTableDesc(final Path sourcePath, final org.apache.hadoop.hive.ql.plan.TableDesc table, - final DynamicPartitionCtx dpCtx) { + final DynamicPartitionCtx dpCtx, + final AcidUtils.Operation writeType) { super(sourcePath); this.dpCtx = dpCtx; if (dpCtx != null && dpCtx.getPartSpec() != null && partitionSpec == null) { - init(table, dpCtx.getPartSpec(), true); + init(table, dpCtx.getPartSpec(), true, writeType); } else { - init(table, new LinkedHashMap(), true); + init(table, new LinkedHashMap(), true, writeType); } } private void init( final org.apache.hadoop.hive.ql.plan.TableDesc table, - final Map partitionSpec, final boolean replace) { + final Map partitionSpec, + final boolean replace, + AcidUtils.Operation writeType) { this.table = table; this.partitionSpec = partitionSpec; this.replace = replace; this.holdDDLTime = false; + this.writeType = writeType; } public void setHoldDDLTime(boolean ddlTime) { @@ -144,4 +181,8 @@ public ListBucketingCtx getLbCtx() { public void setLbCtx(ListBucketingCtx lbCtx) { this.lbCtx = lbCtx; } + + public AcidUtils.Operation getWriteType() { + return writeType; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java index 6dce30c..fdc1f62 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.hooks.ReadEntity; +import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils; import org.apache.hadoop.hive.ql.io.HiveOutputFormat; import org.apache.hadoop.hive.ql.io.HivePassThroughOutputFormat; @@ -597,19 +598,22 @@ public int compare(FieldSchema o1, FieldSchema o2) { * @param numReducers * The number of reducers, set to -1 for automatic inference based on * input data size. + * @param writeType Whether this is an Acid write, and if so whether it is insert, update, + * or delete. * @return The reduceSinkDesc object. */ public static ReduceSinkDesc getReduceSinkDesc( ArrayList keyCols, ArrayList valueCols, List outputColumnNames, boolean includeKeyCols, int tag, - ArrayList partitionCols, String order, int numReducers) { + ArrayList partitionCols, String order, int numReducers, + AcidUtils.Operation writeType) { return getReduceSinkDesc(keyCols, keyCols.size(), valueCols, new ArrayList>(), includeKeyCols ? outputColumnNames.subList(0, keyCols.size()) : new ArrayList(), includeKeyCols ? outputColumnNames.subList(keyCols.size(), outputColumnNames.size()) : outputColumnNames, - includeKeyCols, tag, partitionCols, order, numReducers); + includeKeyCols, tag, partitionCols, order, numReducers, writeType); } /** @@ -635,6 +639,8 @@ public static ReduceSinkDesc getReduceSinkDesc( * @param numReducers * The number of reducers, set to -1 for automatic inference based on * input data size. + * @param writeType Whether this is an Acid write, and if so whether it is insert, update, + * or delete. * @return The reduceSinkDesc object. */ public static ReduceSinkDesc getReduceSinkDesc( @@ -644,7 +650,8 @@ public static ReduceSinkDesc getReduceSinkDesc( List outputKeyColumnNames, List outputValueColumnNames, boolean includeKeyCols, int tag, - ArrayList partitionCols, String order, int numReducers) { + ArrayList partitionCols, String order, int numReducers, + AcidUtils.Operation writeType) { TableDesc keyTable = null; TableDesc valueTable = null; ArrayList outputKeyCols = new ArrayList(); @@ -670,7 +677,7 @@ public static ReduceSinkDesc getReduceSinkDesc( return new ReduceSinkDesc(keyCols, numKeys, valueCols, outputKeyCols, distinctColIndices, outputValCols, tag, partitionCols, numReducers, keyTable, - valueTable); + valueTable, writeType); } /** @@ -690,12 +697,15 @@ public static ReduceSinkDesc getReduceSinkDesc( * @param numReducers * The number of reducers, set to -1 for automatic inference based on * input data size. + * @param writeType Whether this is an Acid write, and if so whether it is insert, update, + * or delete. * @return The reduceSinkDesc object. */ public static ReduceSinkDesc getReduceSinkDesc( ArrayList keyCols, ArrayList valueCols, List outputColumnNames, boolean includeKey, int tag, - int numPartitionFields, int numReducers) throws SemanticException { + int numPartitionFields, int numReducers, AcidUtils.Operation writeType) + throws SemanticException { return getReduceSinkDesc(keyCols, keyCols.size(), valueCols, new ArrayList>(), includeKey ? outputColumnNames.subList(0, keyCols.size()) : @@ -703,7 +713,7 @@ public static ReduceSinkDesc getReduceSinkDesc( includeKey ? outputColumnNames.subList(keyCols.size(), outputColumnNames.size()) : outputColumnNames, - includeKey, tag, numPartitionFields, numReducers); + includeKey, tag, numPartitionFields, numReducers, writeType); } /** @@ -729,6 +739,8 @@ public static ReduceSinkDesc getReduceSinkDesc( * @param numReducers * The number of reducers, set to -1 for automatic inference based on * input data size. + * @param writeType Whether this is an Acid write, and if so whether it is insert, update, + * or delete. * @return The reduceSinkDesc object. */ public static ReduceSinkDesc getReduceSinkDesc( @@ -737,7 +749,8 @@ public static ReduceSinkDesc getReduceSinkDesc( List> distinctColIndices, List outputKeyColumnNames, List outputValueColumnNames, boolean includeKey, int tag, - int numPartitionFields, int numReducers) throws SemanticException { + int numPartitionFields, int numReducers, AcidUtils.Operation writeType) + throws SemanticException { ArrayList partitionCols = new ArrayList(); if (numPartitionFields >= keyCols.size()) { @@ -755,7 +768,7 @@ public static ReduceSinkDesc getReduceSinkDesc( } return getReduceSinkDesc(keyCols, numKeys, valueCols, distinctColIndices, outputKeyColumnNames, outputValueColumnNames, includeKey, tag, - partitionCols, order.toString(), numReducers); + partitionCols, order.toString(), numReducers, writeType); } /** diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java index 5695f35..2c4175a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java @@ -23,6 +23,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.ql.io.AcidUtils; /** @@ -91,6 +92,9 @@ private boolean skipTag; // Skip writing tags when feeding into mapjoin hashtable private Boolean autoParallel = null; // Is reducer auto-parallelism enabled, disabled or unset + // Write type, since this needs to calculate buckets differently for updates and deletes + private AcidUtils.Operation writeType; + private static transient Log LOG = LogFactory.getLog(ReduceSinkDesc.class); public ReduceSinkDesc() { } @@ -102,7 +106,8 @@ public ReduceSinkDesc(ArrayList keyCols, List> distinctColumnIndices, ArrayList outputValueColumnNames, int tag, ArrayList partitionCols, int numReducers, - final TableDesc keySerializeInfo, final TableDesc valueSerializeInfo) { + final TableDesc keySerializeInfo, final TableDesc valueSerializeInfo, + AcidUtils.Operation writeType) { this.keyCols = keyCols; this.numDistributionKeys = numDistributionKeys; this.valueCols = valueCols; @@ -116,6 +121,7 @@ public ReduceSinkDesc(ArrayList keyCols, this.distinctColumnIndices = distinctColumnIndices; this.setNumBuckets(-1); this.setBucketCols(null); + this.writeType = writeType; } @Override @@ -367,4 +373,8 @@ public final void setAutoParallel(final boolean autoParallel) { this.autoParallel = autoParallel; } } + + public AcidUtils.Operation getWriteType() { + return writeType; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index 5164b16..0d57ef3 100644 --- ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -208,6 +208,11 @@ private String hdfsScratchDirURIString; /** + * Next value to use in naming a temporary table created by an insert...values statement + */ + private int nextValueTempTableSuffix = 1; + + /** * Transaction manager to use for this session. This is instantiated lazily by * {@link #initTxnMgr(org.apache.hadoop.hive.conf.HiveConf)} */ @@ -1341,4 +1346,12 @@ public void setUserIpAddress(String userIpAddress) { this.userIpAddress = userIpAddress; } + /** + * Get the next suffix to use in naming a temporary table created by insert...values + * @return suffix + */ + public String getNextValuesTempTableSuffix() { + return Integer.toString(nextValueTempTableSuffix++); + } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java index 789c780..b669754 100755 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.CastDecimalToLong; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.CastDoubleToLong; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.CastTimestampToLongViaLongToLong; +import org.apache.hadoop.hive.ql.io.RecordIdentifier; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; @@ -204,4 +205,19 @@ public IntWritable evaluate(HiveDecimalWritable i) { } } + /** + * Convert a RecordIdentifier. This is done so that we can use the RecordIdentifier in place + * of the bucketing column. + * @param i RecordIdentifier to convert + * @return value of the bucket identifier + */ + public IntWritable evaluate(RecordIdentifier i) { + if (i == null) { + return null; + } else { + intWritable.set(i.getBucketId()); + return intWritable; + } + } + } diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java index 63ecb8d..0eb7c5a 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hive.ql.WindowsPathUtil; import org.apache.hadoop.hive.ql.exec.mr.ExecDriver; import org.apache.hadoop.hive.ql.exec.mr.MapRedTask; +import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.Table; @@ -137,7 +138,7 @@ db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, src, true, true); db.createTable(src, cols, null, TextInputFormat.class, IgnoreKeyTextOutputFormat.class); - db.loadTable(hadoopDataFile[i], src, false, false, true, false); + db.loadTable(hadoopDataFile[i], src, false, false, true, false, false); i++; } @@ -246,7 +247,7 @@ private void populateMapRedPlan1(Table src) throws SemanticException { Operator op1 = OperatorFactory.get(PlanUtils .getReduceSinkDesc(Utilities.makeList(getStringColumn("key")), Utilities.makeList(getStringColumn("value")), outputColumns, true, - -1, 1, -1)); + -1, 1, -1, AcidUtils.Operation.NOT_ACID)); addMapWork(mr, src, "a", op1); ReduceWork rWork = new ReduceWork(); @@ -276,7 +277,7 @@ private void populateMapRedPlan2(Table src) throws SemanticException { .getReduceSinkDesc(Utilities.makeList(getStringColumn("key")), Utilities .makeList(getStringColumn("key"), getStringColumn("value")), - outputColumns, false, -1, 1, -1)); + outputColumns, false, -1, 1, -1, AcidUtils.Operation.NOT_ACID)); addMapWork(mr, src, "a", op1); ReduceWork rWork = new ReduceWork(); @@ -310,14 +311,14 @@ private void populateMapRedPlan3(Table src, Table src2) throws SemanticException Operator op1 = OperatorFactory.get(PlanUtils .getReduceSinkDesc(Utilities.makeList(getStringColumn("key")), Utilities.makeList(getStringColumn("value")), outputColumns, true, - Byte.valueOf((byte) 0), 1, -1)); + Byte.valueOf((byte) 0), 1, -1, AcidUtils.Operation.NOT_ACID)); addMapWork(mr, src, "a", op1); Operator op2 = OperatorFactory.get(PlanUtils .getReduceSinkDesc(Utilities.makeList(getStringColumn("key")), Utilities.makeList(getStringColumn("key")), outputColumns, true, - Byte.valueOf((byte) 1), Integer.MAX_VALUE, -1)); + Byte.valueOf((byte) 1), Integer.MAX_VALUE, -1, AcidUtils.Operation.NOT_ACID)); addMapWork(mr, src2, "b", op2); ReduceWork rWork = new ReduceWork(); @@ -353,7 +354,7 @@ private void populateMapRedPlan4(Table src) throws SemanticException { Operator op1 = OperatorFactory.get(PlanUtils .getReduceSinkDesc(Utilities.makeList(getStringColumn("tkey")), Utilities.makeList(getStringColumn("tkey"), - getStringColumn("tvalue")), outputColumns, false, -1, 1, -1)); + getStringColumn("tvalue")), outputColumns, false, -1, 1, -1, AcidUtils.Operation.NOT_ACID)); Operator op0 = OperatorFactory.get(new ScriptDesc("cat", PlanUtils.getDefaultTableDesc("" + Utilities.tabCode, "key,value"), @@ -398,7 +399,7 @@ private void populateMapRedPlan5(Table src) throws SemanticException { Operator op0 = OperatorFactory.get(PlanUtils .getReduceSinkDesc(Utilities.makeList(getStringColumn("0")), Utilities .makeList(getStringColumn("0"), getStringColumn("1")), - outputColumns, false, -1, 1, -1)); + outputColumns, false, -1, 1, -1, AcidUtils.Operation.NOT_ACID)); Operator op4 = OperatorFactory.get(new SelectDesc(Utilities .makeList(getStringColumn("key"), getStringColumn("value")), @@ -432,7 +433,7 @@ private void populateMapRedPlan6(Table src) throws SemanticException { Operator op1 = OperatorFactory.get(PlanUtils .getReduceSinkDesc(Utilities.makeList(getStringColumn("tkey")), Utilities.makeList(getStringColumn("tkey"), - getStringColumn("tvalue")), outputColumns, false, -1, 1, -1)); + getStringColumn("tvalue")), outputColumns, false, -1, 1, -1, AcidUtils.Operation.NOT_ACID)); Operator op0 = OperatorFactory.get(new ScriptDesc( "\'cat\'", PlanUtils.getDefaultTableDesc("" + Utilities.tabCode, diff --git ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java new file mode 100644 index 0000000..4d1b7df --- /dev/null +++ ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java @@ -0,0 +1,323 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.parse; + +import static org.junit.Assert.*; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import junit.framework.Assert; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.Context; +import org.apache.hadoop.hive.ql.QueryPlan; +import org.apache.hadoop.hive.ql.exec.ExplainTask; +import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; +import org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.plan.ExplainWork; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +public class TestUpdateDeleteSemanticAnalyzer { + + static final private Log LOG = LogFactory.getLog(TestSemanticAnalyzer.class.getName()); + + private HiveConf conf; + private Hive db; + + // All of the insert, update, and delete tests assume two tables, T and U, each with columns a, + // and b. U it partitioned by an additional column ds. These are created by parseAndAnalyze + // and removed by cleanupTables(). + + @Test + public void testInsertSelect() throws Exception { + try { + ReturnInfo rc = parseAndAnalyze("insert into table T select a, b from U", "testInsertSelect"); + + LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump())); + + } finally { + cleanupTables(); + } + } + + @Test + public void testDeleteAllNonPartitioned() throws Exception { + try { + ReturnInfo rc = parseAndAnalyze("delete from T", "testDeleteAllNonPartitioned"); + LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump())); + } finally { + cleanupTables(); + } + } + + @Test + public void testDeleteWhereNoPartition() throws Exception { + try { + ReturnInfo rc = parseAndAnalyze("delete from T where a > 5", "testDeleteWhereNoPartition"); + LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump())); + } finally { + cleanupTables(); + } + } + + @Test + public void testDeleteAllPartitioned() throws Exception { + try { + ReturnInfo rc = parseAndAnalyze("delete from U", "testDeleteAllPartitioned"); + LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump())); + } finally { + cleanupTables(); + } + } + + @Test + public void testDeleteAllWherePartitioned() throws Exception { + try { + ReturnInfo rc = parseAndAnalyze("delete from U where a > 5", "testDeleteAllWherePartitioned"); + LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump())); + } finally { + cleanupTables(); + } + } + + @Test + public void testDeleteOnePartition() throws Exception { + try { + ReturnInfo rc = parseAndAnalyze("delete from U where ds = 'today'", + "testDeleteFromPartitionOnly"); + LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump())); + } finally { + cleanupTables(); + } + } + + @Test + public void testDeleteOnePartitionWhere() throws Exception { + try { + ReturnInfo rc = parseAndAnalyze("delete from U where ds = 'today' and a > 5", + "testDeletePartitionWhere"); + LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump())); + } finally { + cleanupTables(); + } + } + + @Test + public void testUpdateAllNonPartitioned() throws Exception { + try { + ReturnInfo rc = parseAndAnalyze("update T set a = 5", "testUpdateAllNonPartitioned"); + LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump())); + } finally { + cleanupTables(); + } + } + + @Test + public void testUpdateAllNonPartitionedWhere() throws Exception { + try { + ReturnInfo rc = parseAndAnalyze("update T set a = 5 where b > 5", + "testUpdateAllNonPartitionedWhere"); + LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump())); + } finally { + cleanupTables(); + } + } + + @Test + public void testUpdateAllPartitioned() throws Exception { + try { + ReturnInfo rc = parseAndAnalyze("update U set a = 5", "testUpdateAllPartitioned"); + LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump())); + } finally { + cleanupTables(); + } + } + + @Test + public void testUpdateAllPartitionedWhere() throws Exception { + try { + ReturnInfo rc = parseAndAnalyze("update U set a = 5 where b > 5", + "testUpdateAllPartitionedWhere"); + LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump())); + } finally { + cleanupTables(); + } + } + + @Test + public void testUpdateOnePartition() throws Exception { + try { + ReturnInfo rc = parseAndAnalyze("update U set a = 5 where ds = 'today'", + "testUpdateOnePartition"); + LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump())); + } finally { + cleanupTables(); + } + } + + @Test + public void testUpdateOnePartitionWhere() throws Exception { + try { + ReturnInfo rc = parseAndAnalyze("update U set a = 5 where ds = 'today' and b > 5", + "testUpdateOnePartitionWhere"); + LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump())); + } finally { + cleanupTables(); + } + } + + @Test + public void testInsertValues() throws Exception { + try { + ReturnInfo rc = parseAndAnalyze("insert into table T values ('abc', 3), ('ghi', 5)", + "testInsertValues"); + + LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump())); + + } finally { + cleanupTables(); + } + } + + @Test + public void testInsertValuesPartitioned() throws Exception { + try { + ReturnInfo rc = parseAndAnalyze("insert into table U partition (ds) values " + + "('abc', 3, 'today'), ('ghi', 5, 'tomorrow')", + "testInsertValuesPartitioned"); + + LOG.info(explain((SemanticAnalyzer) rc.sem, rc.plan, rc.ast.dump())); + + } finally { + cleanupTables(); + } + } + + @Before + public void setup() { + conf = new HiveConf(); + conf.setVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict"); + conf.setVar(HiveConf.ConfVars.HIVE_TXN_MANAGER, "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager"); + } + + public void cleanupTables() throws HiveException { + if (db != null) { + db.dropTable("T"); + db.dropTable("U"); + } + } + + private class ReturnInfo { + ASTNode ast; + BaseSemanticAnalyzer sem; + QueryPlan plan; + + ReturnInfo(ASTNode a, BaseSemanticAnalyzer s, QueryPlan p) { + ast = a; + sem = s; + plan = p; + } + } + + private ReturnInfo parseAndAnalyze(String query, String testName) + throws IOException, ParseException, HiveException { + + SessionState.start(conf); + Context ctx = new Context(conf); + ctx.setCmd(query); + ctx.setHDFSCleanup(true); + + ParseDriver pd = new ParseDriver(); + ASTNode tree = pd.parse(query, ctx); + tree = ParseUtils.findRootNonNullToken(tree); + + BaseSemanticAnalyzer sem = SemanticAnalyzerFactory.get(conf, tree); + SessionState.get().initTxnMgr(conf); + db = sem.getDb(); + + // I have to create the tables here (rather than in setup()) because I need the Hive + // connection, which is conviently created by the semantic analyzer. + db.createTable("T", Arrays.asList("a", "b"), null, OrcInputFormat.class, OrcOutputFormat.class); + db.createTable("U", Arrays.asList("a", "b"), Arrays.asList("ds"), OrcInputFormat.class, + OrcOutputFormat.class); + Table u = db.getTable("U"); + Map partVals = new HashMap(2); + partVals.put("ds", "yesterday"); + db.createPartition(u, partVals); + partVals.clear(); + partVals.put("ds", "today"); + db.createPartition(u, partVals); + sem.analyze(tree, ctx); + // validate the plan + sem.validate(); + + QueryPlan plan = new QueryPlan(query, sem, 0L, testName); + + return new ReturnInfo(tree, sem, plan); + } + + private String explain(SemanticAnalyzer sem, QueryPlan plan, String astStringTree) throws + IOException { + FileSystem fs = FileSystem.get(conf); + File f = File.createTempFile("TestSemanticAnalyzer", "explain"); + Path tmp = new Path(f.getPath()); + fs.create(tmp); + fs.deleteOnExit(tmp); + ExplainWork work = new ExplainWork(tmp, sem.getParseContext(), sem.getRootTasks(), + sem.getFetchTask(), astStringTree, sem, true, false, false, false, false); + ExplainTask task = new ExplainTask(); + task.setWork(work); + task.initialize(conf, plan, null); + task.execute(null); + FSDataInputStream in = fs.open(tmp); + StringBuilder builder = new StringBuilder(); + final int bufSz = 4096; + byte[] buf = new byte[bufSz]; + long pos = 0L; + while (true) { + int bytesRead = in.read(pos, buf, 0, bufSz); + if (bytesRead > 0) { + pos += bytesRead; + builder.append(new String(buf, 0, bytesRead)); + } else { + // Reached end of file + in.close(); + break; + } + } + return builder.toString() + .replaceAll("pfile:/.*\n", "pfile:MASKED-OUT\n") + .replaceAll("location file:/.*\n", "location file:MASKED-OUT\n") + .replaceAll("file:/.*\n", "file:MASKED-OUT\n") + .replaceAll("transient_lastDdlTime.*\n", "transient_lastDdlTime MASKED-OUT\n"); + } +} diff --git ql/src/test/queries/clientnegative/acid_overwrite.q ql/src/test/queries/clientnegative/acid_overwrite.q new file mode 100644 index 0000000..9f6c1f8 --- /dev/null +++ ql/src/test/queries/clientnegative/acid_overwrite.q @@ -0,0 +1,9 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +set hive.enforce.bucketing=true; + +create table acid_uanp(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc; + +insert into table acid_uanp select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint < 0 order by cint limit 10; +insert overwrite table acid_uanp select cint, cast(cstring1 as varchar(128)) from alltypesorc; diff --git ql/src/test/queries/clientnegative/delete_not_acid.q ql/src/test/queries/clientnegative/delete_not_acid.q new file mode 100644 index 0000000..ae24418 --- /dev/null +++ ql/src/test/queries/clientnegative/delete_not_acid.q @@ -0,0 +1,6 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager; + +create table foo(a int, b varchar(128)) clustered by (a) into 1 buckets stored as orc; + +delete from foo; diff --git ql/src/test/queries/clientnegative/update_not_acid.q ql/src/test/queries/clientnegative/update_not_acid.q new file mode 100644 index 0000000..ac9f4ad --- /dev/null +++ ql/src/test/queries/clientnegative/update_not_acid.q @@ -0,0 +1,6 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager; + +create table foo(a int, b varchar(128)) clustered by (a) into 1 buckets stored as orc; + +update foo set b = 'fred'; diff --git ql/src/test/queries/clientnegative/update_partition_col.q ql/src/test/queries/clientnegative/update_partition_col.q new file mode 100644 index 0000000..918d312 --- /dev/null +++ ql/src/test/queries/clientnegative/update_partition_col.q @@ -0,0 +1,8 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +set hive.enforce.bucketing=true; + +create table foo(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc; + +update foo set ds = 'fred'; diff --git ql/src/test/queries/clientpositive/delete_all_non_partitioned.q ql/src/test/queries/clientpositive/delete_all_non_partitioned.q new file mode 100644 index 0000000..3c0bf62 --- /dev/null +++ ql/src/test/queries/clientpositive/delete_all_non_partitioned.q @@ -0,0 +1,17 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +set hive.enforce.bucketing=true; +set hive.exec.reducers.max = 1; + +create table acid_danp(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc; + +insert into table acid_danp select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint < 0 order by cint limit 10; + +select a,b from acid_danp order by a; + +delete from acid_danp; + +select a,b from acid_danp; + + diff --git ql/src/test/queries/clientpositive/delete_all_partitioned.q ql/src/test/queries/clientpositive/delete_all_partitioned.q new file mode 100644 index 0000000..c271896 --- /dev/null +++ ql/src/test/queries/clientpositive/delete_all_partitioned.q @@ -0,0 +1,16 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +set hive.enforce.bucketing=true; +set hive.mapred.supports.subdirectories=true; + +create table acid_dap(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc; + +insert into table acid_dap partition (ds='today') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint < 0 order by cint limit 10; +insert into table acid_dap partition (ds='tomorrow') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint > 1000 order by cint limit 10; + +select a,b,ds from acid_dap order by a,b; + +delete from acid_dap; + +select * from acid_dap; diff --git ql/src/test/queries/clientpositive/delete_orig_table.q ql/src/test/queries/clientpositive/delete_orig_table.q new file mode 100644 index 0000000..e1759f6 --- /dev/null +++ ql/src/test/queries/clientpositive/delete_orig_table.q @@ -0,0 +1,29 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +set hive.enforce.bucketing=true; + +dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/delete_orig_table; +dfs -copyFromLocal ../../data/files/alltypesorc ${system:test.tmp.dir}/delete_orig_table/00000_0; + +create table acid_dot( + ctinyint TINYINT, + csmallint SMALLINT, + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE, + cstring1 STRING, + cstring2 STRING, + ctimestamp1 TIMESTAMP, + ctimestamp2 TIMESTAMP, + cboolean1 BOOLEAN, + cboolean2 BOOLEAN) clustered by (cint) into 1 buckets stored as orc location '${system:test.tmp.dir}/delete_orig_table'; + +select count(*) from acid_dot; + +delete from acid_dot where cint < -1070551679; + +select count(*) from acid_dot; + +dfs -rmr ${system:test.tmp.dir}/delete_orig_table; diff --git ql/src/test/queries/clientpositive/delete_tmp_table.q ql/src/test/queries/clientpositive/delete_tmp_table.q new file mode 100644 index 0000000..5563b3c --- /dev/null +++ ql/src/test/queries/clientpositive/delete_tmp_table.q @@ -0,0 +1,16 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +set hive.enforce.bucketing=true; + +create temporary table acid_dtt(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc; + +insert into table acid_dtt select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null order by cint limit 10; + +select * from acid_dtt order by a; + +delete from acid_dtt where b = '0ruyd6Y50JpdGRf6HqD' or b = '2uLyD28144vklju213J1mr'; + +select a,b from acid_dtt order by b; + + diff --git ql/src/test/queries/clientpositive/delete_where_no_match.q ql/src/test/queries/clientpositive/delete_where_no_match.q new file mode 100644 index 0000000..8ebff45 --- /dev/null +++ ql/src/test/queries/clientpositive/delete_where_no_match.q @@ -0,0 +1,16 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +set hive.enforce.bucketing=true; + +create table acid_dwnm(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc; + +insert into table acid_dwnm select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null order by cint limit 10; + +select * from acid_dwnm order by a; + +delete from acid_dwnm where b = 'nosuchvalue'; + +select a,b from acid_dwnm order by b; + + diff --git ql/src/test/queries/clientpositive/delete_where_non_partitioned.q ql/src/test/queries/clientpositive/delete_where_non_partitioned.q new file mode 100644 index 0000000..b37ec80 --- /dev/null +++ ql/src/test/queries/clientpositive/delete_where_non_partitioned.q @@ -0,0 +1,16 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +set hive.enforce.bucketing=true; + +create table acid_dwnp(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc; + +insert into table acid_dwnp select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null order by cint limit 10; + +select * from acid_dwnp order by a; + +delete from acid_dwnp where b = '0ruyd6Y50JpdGRf6HqD'; + +select a,b from acid_dwnp order by b; + + diff --git ql/src/test/queries/clientpositive/delete_where_partitioned.q ql/src/test/queries/clientpositive/delete_where_partitioned.q new file mode 100644 index 0000000..04dc03e --- /dev/null +++ ql/src/test/queries/clientpositive/delete_where_partitioned.q @@ -0,0 +1,16 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +set hive.enforce.bucketing=true; +set hive.mapred.supports.subdirectories=true; + +create table acid_dwp(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc; + +insert into table acid_dwp partition (ds='today') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint < 0 order by cint limit 10; +insert into table acid_dwp partition (ds='tomorrow') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint > -10000000 order by cint limit 10; + +select a,b,ds from acid_dwp order by a, ds; + +delete from acid_dwp where a = '-1071363017'; + +select * from acid_dwp order by a, ds; diff --git ql/src/test/queries/clientpositive/delete_whole_partition.q ql/src/test/queries/clientpositive/delete_whole_partition.q new file mode 100644 index 0000000..4548cb6 --- /dev/null +++ ql/src/test/queries/clientpositive/delete_whole_partition.q @@ -0,0 +1,16 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +set hive.enforce.bucketing=true; +set hive.mapred.supports.subdirectories=true; + +create table acid_dwhp(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc; + +insert into table acid_dwhp partition (ds='today') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint < 0 order by cint limit 10; +insert into table acid_dwhp partition (ds='tomorrow') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint > -10000000 order by cint limit 10; + +select a,b,ds from acid_dwhp order by a, ds; + +delete from acid_dwhp where ds = 'today'; + +select * from acid_dwhp order by a, ds; diff --git ql/src/test/queries/clientpositive/insert_orig_table.q ql/src/test/queries/clientpositive/insert_orig_table.q new file mode 100644 index 0000000..2c6df88 --- /dev/null +++ ql/src/test/queries/clientpositive/insert_orig_table.q @@ -0,0 +1,28 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +set hive.enforce.bucketing=true; + +create table acid_iot( + ctinyint TINYINT, + csmallint SMALLINT, + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE, + cstring1 STRING, + cstring2 STRING, + ctimestamp1 TIMESTAMP, + ctimestamp2 TIMESTAMP, + cboolean1 BOOLEAN, + cboolean2 BOOLEAN) clustered by (cint) into 1 buckets stored as orc; + +LOAD DATA LOCAL INPATH "../../data/files/alltypesorc" into table acid_iot; + +select count(*) from acid_iot; + +insert into table acid_iot select ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, + cboolean1, cboolean2 from alltypesorc where cint < 0 order by cint limit 10; + +select count(*) from acid_iot; + diff --git ql/src/test/queries/clientpositive/insert_update_delete.q ql/src/test/queries/clientpositive/insert_update_delete.q new file mode 100644 index 0000000..bbae4e9 --- /dev/null +++ ql/src/test/queries/clientpositive/insert_update_delete.q @@ -0,0 +1,20 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +set hive.enforce.bucketing=true; +set hive.mapred.supports.subdirectories=true; + +create table acid_iud(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc; + +insert into table acid_iud select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint < 0 order by cint limit 10; + +select a,b from acid_iud order by a; + +update acid_iud set b = 'fred'; + +select a,b from acid_iud order by a; + +delete from acid_iud; + +select a,b from acid_iud order by a; + diff --git ql/src/test/queries/clientpositive/insert_values_dynamic_partitioned.q ql/src/test/queries/clientpositive/insert_values_dynamic_partitioned.q new file mode 100644 index 0000000..2602189 --- /dev/null +++ ql/src/test/queries/clientpositive/insert_values_dynamic_partitioned.q @@ -0,0 +1,16 @@ +set hive.exec.dynamic.partition.mode=nonstrict; +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +set hive.enforce.bucketing=true; +set hive.mapred.supports.subdirectories=true; + +create table ivdp(i int, + de decimal(5,2), + vc varchar(128)) partitioned by (ds string) clustered by (i) into 2 buckets stored as orc; + +insert into table ivdp partition (ds) values + (1, 109.23, 'and everywhere that mary went', 'today'), + (6553, 923.19, 'the lamb was sure to go', 'tomorrow'); + +select * from ivdp order by ds; diff --git ql/src/test/queries/clientpositive/insert_values_non_partitioned.q ql/src/test/queries/clientpositive/insert_values_non_partitioned.q new file mode 100644 index 0000000..6edfd6a --- /dev/null +++ ql/src/test/queries/clientpositive/insert_values_non_partitioned.q @@ -0,0 +1,23 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +set hive.enforce.bucketing=true; + +create table acid_ivnp(ti tinyint, + si smallint, + i int, + bi bigint, + f float, + d double, + de decimal(5,2), + t timestamp, + dt date, + s string, + vc varchar(128), + ch char(12)) clustered by (i) into 2 buckets stored as orc; + +insert into table acid_ivnp values + (1, 257, 65537, 4294967297, 3.14, 3.141592654, 109.23, '2014-08-25 17:21:30.0', '2014-08-25', 'mary had a little lamb', 'ring around the rosie', 'red'), + (3, 25, 6553, 429496729, 0.14, 1923.141592654, 1.2301, '2014-08-24 17:21:30.0', '2014-08-26', 'its fleece was white as snow', 'a pocket full of posies', 'blue' ); + +select ti, si, i, bi, f, d, de, t, dt, s, vc, ch from acid_ivnp order by ti; diff --git ql/src/test/queries/clientpositive/insert_values_orig_table.q ql/src/test/queries/clientpositive/insert_values_orig_table.q new file mode 100644 index 0000000..d3d68d2 --- /dev/null +++ ql/src/test/queries/clientpositive/insert_values_orig_table.q @@ -0,0 +1,29 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +set hive.enforce.bucketing=true; + +create table acid_ivot( + ctinyint TINYINT, + csmallint SMALLINT, + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE, + cstring1 STRING, + cstring2 STRING, + ctimestamp1 TIMESTAMP, + ctimestamp2 TIMESTAMP, + cboolean1 BOOLEAN, + cboolean2 BOOLEAN) clustered by (cint) into 1 buckets stored as orc; + +LOAD DATA LOCAL INPATH "../../data/files/alltypesorc" into table acid_ivot; + +select count(*) from acid_ivot; + +insert into table acid_ivot values + (1, 2, 3, 4, 3.14, 2.34, 'fred', 'bob', '2014-09-01 10:34:23.111', '1944-06-06 06:00:00', true, true), + (111, 222, 3333, 444, 13.14, 10239302.34239320, 'fred', 'bob', '2014-09-01 10:34:23.111', '1944-06-06 06:00:00', true, true); + +select count(*) from acid_ivot; + diff --git ql/src/test/queries/clientpositive/insert_values_partitioned.q ql/src/test/queries/clientpositive/insert_values_partitioned.q new file mode 100644 index 0000000..11f0d4e --- /dev/null +++ ql/src/test/queries/clientpositive/insert_values_partitioned.q @@ -0,0 +1,24 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +set hive.enforce.bucketing=true; +set hive.exec.dynamic.partition.mode=nonstrict; + +create table acid_ivp(ti tinyint, + si smallint, + i int, + bi bigint, + f float, + d double, + de decimal(5,2), + t timestamp, + dt date, + s string, + vc varchar(128), + ch char(12)) partitioned by (ds string) clustered by (i) into 2 buckets stored as orc; + +insert into table acid_ivp partition (ds='today') values + (1, 257, 65537, 4294967297, 3.14, 3.141592654, 109.23, '2014-08-25 17:21:30.0', '2014-08-25', 'mary had a little lamb', 'ring around the rosie', 'red'), + (3, 25, 6553, 429496729, 0.14, 1923.141592654, 1.2301, '2014-08-24 17:21:30.0', '2014-08-26', 'its fleece was white as snow', 'a pocket full of posies', 'blue'); + +select * from acid_ivp order by i; diff --git ql/src/test/queries/clientpositive/insert_values_tmp_table.q ql/src/test/queries/clientpositive/insert_values_tmp_table.q new file mode 100644 index 0000000..fd8ec29 --- /dev/null +++ ql/src/test/queries/clientpositive/insert_values_tmp_table.q @@ -0,0 +1,12 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +set hive.enforce.bucketing=true; + +create temporary table acid_ivtt(i int, de decimal(5,2), vc varchar(128)) clustered by (vc) into 2 buckets stored as orc; + +insert into table acid_ivtt values + (1, 109.23, 'mary had a little lamb'), + (429496729, 0.14, 'its fleece was white as snow'); + +select i, de, vc from acid_ivtt order by i; diff --git ql/src/test/queries/clientpositive/update_after_multiple_inserts.q ql/src/test/queries/clientpositive/update_after_multiple_inserts.q new file mode 100644 index 0000000..f7aab4d --- /dev/null +++ ql/src/test/queries/clientpositive/update_after_multiple_inserts.q @@ -0,0 +1,24 @@ +set hive.exec.dynamic.partition.mode=nonstrict; +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +set hive.enforce.bucketing=true; +set hive.mapred.supports.subdirectories=true; + +create table acid_uami(i int, + de decimal(5,2), + vc varchar(128)) clustered by (i) into 2 buckets stored as orc; + +insert into table acid_uami values + (1, 109.23, 'mary had a little lamb'), + (6553, 923.19, 'its fleece was white as snow'); + +insert into table acid_uami values + (10, 119.23, 'and everywhere that mary went'), + (65530, 823.19, 'the lamb was sure to go'); + +select * from acid_uami order by de; + +update acid_uami set de = 3.14 where de = 109.23 or de = 119.23; + +select * from acid_uami order by de; diff --git ql/src/test/queries/clientpositive/update_all_non_partitioned.q ql/src/test/queries/clientpositive/update_all_non_partitioned.q new file mode 100644 index 0000000..67d6ba9 --- /dev/null +++ ql/src/test/queries/clientpositive/update_all_non_partitioned.q @@ -0,0 +1,16 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +set hive.enforce.bucketing=true; + +create table acid_uanp(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc; + +insert into table acid_uanp select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint < 0 order by cint limit 10; + +select a,b from acid_uanp order by a; + +update acid_uanp set b = 'fred'; + +select a,b from acid_uanp order by a; + + diff --git ql/src/test/queries/clientpositive/update_all_partitioned.q ql/src/test/queries/clientpositive/update_all_partitioned.q new file mode 100644 index 0000000..9a5870a --- /dev/null +++ ql/src/test/queries/clientpositive/update_all_partitioned.q @@ -0,0 +1,18 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +set hive.enforce.bucketing=true; +set hive.mapred.supports.subdirectories=true; + +create table acid_uap(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc; + +insert into table acid_uap partition (ds='today') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint < 0 order by cint limit 10; +insert into table acid_uap partition (ds='tomorrow') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint > 10 order by cint limit 10; + +select a,b,ds from acid_uap order by a,b; + +update acid_uap set b = 'fred'; + +select a,b,ds from acid_uap order by a,b; + + diff --git ql/src/test/queries/clientpositive/update_all_types.q ql/src/test/queries/clientpositive/update_all_types.q new file mode 100644 index 0000000..39fe73d --- /dev/null +++ ql/src/test/queries/clientpositive/update_all_types.q @@ -0,0 +1,56 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +set hive.enforce.bucketing=true; + +create table acid_uat(ti tinyint, + si smallint, + i int, + bi bigint, + f float, + d double, + de decimal(5,2), + t timestamp, + dt date, + s string, + vc varchar(128), + ch char(36), + b boolean) clustered by (i) into 2 buckets stored as orc; + +insert into table acid_uat + select ctinyint, + csmallint, + cint, + cbigint, + cfloat, + cdouble, + cast(cfloat as decimal(5,2)), + ctimestamp1, + cast(ctimestamp2 as date), + cstring1, + cast(cstring1 as varchar(128)), + cast(cstring2 as char(36)), + cboolean1 + from alltypesorc where cint < 0 order by cint limit 10; + +select * from acid_uat order by i; + +update acid_uat set + ti = 1, + si = 2, + i = 3, + bi = 4, + f = 3.14, + d = 6.28, + de = 5.99, + t = '2014-09-01 09:44.23.23', + dt = '2014-09-01', + s = 'its a beautiful day in the neighbhorhood', + vc = 'a beautiful day for a neighbor', + ch = 'wont you be mine', + b = true + where s = '0ruyd6Y50JpdGRf6HqD'; + +select * from acid_uat order by i; + + diff --git ql/src/test/queries/clientpositive/update_orig_table.q ql/src/test/queries/clientpositive/update_orig_table.q new file mode 100644 index 0000000..f09ad32 --- /dev/null +++ ql/src/test/queries/clientpositive/update_orig_table.q @@ -0,0 +1,27 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +set hive.enforce.bucketing=true; + +dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/update_orig_table; +dfs -copyFromLocal ../../data/files/alltypesorc ${system:test.tmp.dir}/update_orig_table/00000_0; + +create table acid_uot( + ctinyint TINYINT, + csmallint SMALLINT, + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE, + cstring1 STRING, + cstring2 STRING, + ctimestamp1 TIMESTAMP, + ctimestamp2 TIMESTAMP, + cboolean1 BOOLEAN, + cboolean2 BOOLEAN) clustered by (cint) into 1 buckets stored as orc location '${system:test.tmp.dir}/update_orig_table'; + +update acid_uot set cstring1 = 'fred' where cint < -1070551679; + +select * from acid_uot where cstring1 = 'fred'; + +dfs -rmr ${system:test.tmp.dir}/update_orig_table; diff --git ql/src/test/queries/clientpositive/update_tmp_table.q ql/src/test/queries/clientpositive/update_tmp_table.q new file mode 100644 index 0000000..c863cd6 --- /dev/null +++ ql/src/test/queries/clientpositive/update_tmp_table.q @@ -0,0 +1,16 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +set hive.enforce.bucketing=true; + +create table acid_utt(a int, b varchar(128)) clustered by (b) into 2 buckets stored as orc; + +insert into table acid_utt select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null order by cint limit 10; + +select a,b from acid_utt order by a; + +update acid_utt set b = 'fred' where b = '0ruyd6Y50JpdGRf6HqD'; + +select * from acid_utt order by a; + + diff --git ql/src/test/queries/clientpositive/update_two_cols.q ql/src/test/queries/clientpositive/update_two_cols.q new file mode 100644 index 0000000..3233d2f --- /dev/null +++ ql/src/test/queries/clientpositive/update_two_cols.q @@ -0,0 +1,16 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +set hive.enforce.bucketing=true; + +create table acid_utc(a int, b varchar(128), c float) clustered by (a) into 2 buckets stored as orc; + +insert into table acid_utc select cint, cast(cstring1 as varchar(128)), cfloat from alltypesorc where cint < 0 order by cint limit 10; + +select * from acid_utc order by a; + +update acid_utc set b = 'fred',c = 3.14; + +select * from acid_utc order by a; + + diff --git ql/src/test/queries/clientpositive/update_where_no_match.q ql/src/test/queries/clientpositive/update_where_no_match.q new file mode 100644 index 0000000..00583c3 --- /dev/null +++ ql/src/test/queries/clientpositive/update_where_no_match.q @@ -0,0 +1,16 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +set hive.enforce.bucketing=true; + +create table acid_wnm(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc; + +insert into table acid_wnm select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null order by cint limit 10; + +select a,b from acid_wnm order by a; + +update acid_wnm set b = 'fred' where b = 'nosuchvalue'; + +select * from acid_wnm order by a; + + diff --git ql/src/test/queries/clientpositive/update_where_non_partitioned.q ql/src/test/queries/clientpositive/update_where_non_partitioned.q new file mode 100644 index 0000000..378cf94 --- /dev/null +++ ql/src/test/queries/clientpositive/update_where_non_partitioned.q @@ -0,0 +1,16 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +set hive.enforce.bucketing=true; + +create table acid_uwnp(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc; + +insert into table acid_uwnp select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null order by cint limit 10; + +select a,b from acid_uwnp order by a; + +update acid_uwnp set b = 'fred' where b = '0ruyd6Y50JpdGRf6HqD'; + +select * from acid_uwnp order by a; + + diff --git ql/src/test/queries/clientpositive/update_where_partitioned.q ql/src/test/queries/clientpositive/update_where_partitioned.q new file mode 100644 index 0000000..eec745a --- /dev/null +++ ql/src/test/queries/clientpositive/update_where_partitioned.q @@ -0,0 +1,18 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +set hive.enforce.bucketing=true; +set hive.mapred.supports.subdirectories=true; + +create table acid_uwp(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc; + +insert into table acid_uwp partition (ds='today') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint < 0 order by cint limit 10; +insert into table acid_uwp partition (ds='tomorrow') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint > 100 order by cint limit 10; + +select a,b,ds from acid_uwp order by a, ds, b; + +update acid_uwp set b = 'fred' where b = 'k17Am8uPHWk02cEf1jet'; + +select * from acid_uwp order by a, ds, b; + + diff --git ql/src/test/results/clientnegative/acid_overwrite.q.out ql/src/test/results/clientnegative/acid_overwrite.q.out new file mode 100644 index 0000000..f28e0a4 --- /dev/null +++ ql/src/test/results/clientnegative/acid_overwrite.q.out @@ -0,0 +1,19 @@ +PREHOOK: query: create table acid_uanp(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_uanp +POSTHOOK: query: create table acid_uanp(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_uanp +PREHOOK: query: insert into table acid_uanp select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint < 0 order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_uanp +POSTHOOK: query: insert into table acid_uanp select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint < 0 order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_uanp +POSTHOOK: Lineage: acid_uanp.a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_uanp.b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +FAILED: SemanticException [Error 10295]: INSERT OVERWRITE not allowed on table with OutputFormat that implements AcidOutputFormat while transaction manager that supports ACID is in use diff --git ql/src/test/results/clientnegative/delete_not_acid.q.out ql/src/test/results/clientnegative/delete_not_acid.q.out new file mode 100644 index 0000000..24402f1 --- /dev/null +++ ql/src/test/results/clientnegative/delete_not_acid.q.out @@ -0,0 +1,9 @@ +PREHOOK: query: create table foo(a int, b varchar(128)) clustered by (a) into 1 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@foo +POSTHOOK: query: create table foo(a int, b varchar(128)) clustered by (a) into 1 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@foo +FAILED: SemanticException [Error 10294]: Attempt to do update or delete using transaction manager that does not support these operations. diff --git ql/src/test/results/clientnegative/invalid_cast_from_binary_1.q.out ql/src/test/results/clientnegative/invalid_cast_from_binary_1.q.out index f2db9d2..acecbae 100644 --- ql/src/test/results/clientnegative/invalid_cast_from_binary_1.q.out +++ ql/src/test/results/clientnegative/invalid_cast_from_binary_1.q.out @@ -6,4 +6,4 @@ POSTHOOK: query: create table tbl (a binary) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@tbl -FAILED: SemanticException Line 0:-1 Wrong arguments 'a': No matching method for class org.apache.hadoop.hive.ql.udf.UDFToInteger with (binary). Possible choices: _FUNC_(bigint) _FUNC_(boolean) _FUNC_(decimal(38,18)) _FUNC_(double) _FUNC_(float) _FUNC_(smallint) _FUNC_(string) _FUNC_(timestamp) _FUNC_(tinyint) _FUNC_(void) +FAILED: SemanticException Line 0:-1 Wrong arguments 'a': No matching method for class org.apache.hadoop.hive.ql.udf.UDFToInteger with (binary). Possible choices: _FUNC_(bigint) _FUNC_(boolean) _FUNC_(decimal(38,18)) _FUNC_(double) _FUNC_(float) _FUNC_(smallint) _FUNC_(string) _FUNC_(struct) _FUNC_(timestamp) _FUNC_(tinyint) _FUNC_(void) diff --git ql/src/test/results/clientnegative/update_not_acid.q.out ql/src/test/results/clientnegative/update_not_acid.q.out new file mode 100644 index 0000000..24402f1 --- /dev/null +++ ql/src/test/results/clientnegative/update_not_acid.q.out @@ -0,0 +1,9 @@ +PREHOOK: query: create table foo(a int, b varchar(128)) clustered by (a) into 1 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@foo +POSTHOOK: query: create table foo(a int, b varchar(128)) clustered by (a) into 1 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@foo +FAILED: SemanticException [Error 10294]: Attempt to do update or delete using transaction manager that does not support these operations. diff --git ql/src/test/results/clientnegative/update_partition_col.q.out ql/src/test/results/clientnegative/update_partition_col.q.out new file mode 100644 index 0000000..003b53f --- /dev/null +++ ql/src/test/results/clientnegative/update_partition_col.q.out @@ -0,0 +1,9 @@ +PREHOOK: query: create table foo(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@foo +POSTHOOK: query: create table foo(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@foo +FAILED: SemanticException [Error 10292]: Updating values of partition columns is not supported diff --git ql/src/test/results/clientpositive/delete_all_non_partitioned.q.out ql/src/test/results/clientpositive/delete_all_non_partitioned.q.out new file mode 100644 index 0000000..0d428ca --- /dev/null +++ ql/src/test/results/clientpositive/delete_all_non_partitioned.q.out @@ -0,0 +1,52 @@ +PREHOOK: query: create table acid_danp(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_danp +POSTHOOK: query: create table acid_danp(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_danp +PREHOOK: query: insert into table acid_danp select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint < 0 order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_danp +POSTHOOK: query: insert into table acid_danp select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint < 0 order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_danp +POSTHOOK: Lineage: acid_danp.a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_danp.b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: select a,b from acid_danp order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_danp +#### A masked pattern was here #### +POSTHOOK: query: select a,b from acid_danp order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_danp +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa +-1073051226 A34p7oRr2WvUJNf +-1072910839 0iqrc5 +-1072081801 dPkN74F7 +-1072076362 2uLyD28144vklju213J1mr +-1071480828 aw724t8c5558x2xneC624 +-1071363017 Anj0oF +-1070883071 0ruyd6Y50JpdGRf6HqD +-1070551679 iUR3Q +-1069736047 k17Am8uPHWk02cEf1jet +PREHOOK: query: delete from acid_danp +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_danp +PREHOOK: Output: default@acid_danp +POSTHOOK: query: delete from acid_danp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_danp +POSTHOOK: Output: default@acid_danp +PREHOOK: query: select a,b from acid_danp +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_danp +#### A masked pattern was here #### +POSTHOOK: query: select a,b from acid_danp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_danp +#### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/delete_all_partitioned.q.out ql/src/test/results/clientpositive/delete_all_partitioned.q.out new file mode 100644 index 0000000..4486323 --- /dev/null +++ ql/src/test/results/clientpositive/delete_all_partitioned.q.out @@ -0,0 +1,86 @@ +PREHOOK: query: create table acid_dap(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_dap +POSTHOOK: query: create table acid_dap(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_dap +PREHOOK: query: insert into table acid_dap partition (ds='today') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint < 0 order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_dap@ds=today +POSTHOOK: query: insert into table acid_dap partition (ds='today') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint < 0 order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_dap@ds=today +POSTHOOK: Lineage: acid_dap PARTITION(ds=today).a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_dap PARTITION(ds=today).b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: insert into table acid_dap partition (ds='tomorrow') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint > 1000 order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_dap@ds=tomorrow +POSTHOOK: query: insert into table acid_dap partition (ds='tomorrow') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint > 1000 order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_dap@ds=tomorrow +POSTHOOK: Lineage: acid_dap PARTITION(ds=tomorrow).a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_dap PARTITION(ds=tomorrow).b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: select a,b,ds from acid_dap order by a,b +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dap +PREHOOK: Input: default@acid_dap@ds=today +PREHOOK: Input: default@acid_dap@ds=tomorrow +#### A masked pattern was here #### +POSTHOOK: query: select a,b,ds from acid_dap order by a,b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dap +POSTHOOK: Input: default@acid_dap@ds=today +POSTHOOK: Input: default@acid_dap@ds=tomorrow +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa today +-1073051226 A34p7oRr2WvUJNf today +-1072910839 0iqrc5 today +-1072081801 dPkN74F7 today +-1072076362 2uLyD28144vklju213J1mr today +-1071480828 aw724t8c5558x2xneC624 today +-1071363017 Anj0oF today +-1070883071 0ruyd6Y50JpdGRf6HqD today +-1070551679 iUR3Q today +-1069736047 k17Am8uPHWk02cEf1jet today +6981 NULL tomorrow +6981 1FNNhmiFLGw425NA13g tomorrow +6981 4KhrrQ0nJ7bMNTvhSCA tomorrow +6981 K630vaVf tomorrow +6981 Y5x3JuI3M8jngv5N tomorrow +6981 YdG61y00526u5 tomorrow +6981 a3EhVU6Wuy7ycJ7wY7h2gv tomorrow +6981 o4lvY20511w0EOX3P3I82p63 tomorrow +6981 o5mb0QP5Y48Qd4vdB0 tomorrow +6981 sF2CRfgt2K tomorrow +PREHOOK: query: delete from acid_dap +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dap +PREHOOK: Input: default@acid_dap@ds=today +PREHOOK: Input: default@acid_dap@ds=tomorrow +PREHOOK: Output: default@acid_dap@ds=today +PREHOOK: Output: default@acid_dap@ds=tomorrow +POSTHOOK: query: delete from acid_dap +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dap +POSTHOOK: Input: default@acid_dap@ds=today +POSTHOOK: Input: default@acid_dap@ds=tomorrow +POSTHOOK: Output: default@acid_dap@ds=today +POSTHOOK: Output: default@acid_dap@ds=tomorrow +PREHOOK: query: select * from acid_dap +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dap +PREHOOK: Input: default@acid_dap@ds=today +PREHOOK: Input: default@acid_dap@ds=tomorrow +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_dap +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dap +POSTHOOK: Input: default@acid_dap@ds=today +POSTHOOK: Input: default@acid_dap@ds=tomorrow +#### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/delete_orig_table.q.out ql/src/test/results/clientpositive/delete_orig_table.q.out new file mode 100644 index 0000000..a036b06 --- /dev/null +++ ql/src/test/results/clientpositive/delete_orig_table.q.out @@ -0,0 +1,61 @@ +PREHOOK: query: create table acid_dot( + ctinyint TINYINT, + csmallint SMALLINT, + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE, + cstring1 STRING, + cstring2 STRING, + ctimestamp1 TIMESTAMP, + ctimestamp2 TIMESTAMP, + cboolean1 BOOLEAN, +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_dot +POSTHOOK: query: create table acid_dot( + ctinyint TINYINT, + csmallint SMALLINT, + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE, + cstring1 STRING, + cstring2 STRING, + ctimestamp1 TIMESTAMP, + ctimestamp2 TIMESTAMP, + cboolean1 BOOLEAN, +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_dot +PREHOOK: query: select count(*) from acid_dot +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dot +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid_dot +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dot +#### A masked pattern was here #### +12288 +PREHOOK: query: delete from acid_dot where cint < -1070551679 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dot +PREHOOK: Output: default@acid_dot +POSTHOOK: query: delete from acid_dot where cint < -1070551679 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dot +POSTHOOK: Output: default@acid_dot +PREHOOK: query: select count(*) from acid_dot +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dot +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid_dot +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dot +#### A masked pattern was here #### +12280 +#### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/delete_tmp_table.q.out ql/src/test/results/clientpositive/delete_tmp_table.q.out new file mode 100644 index 0000000..ca568b3 --- /dev/null +++ ql/src/test/results/clientpositive/delete_tmp_table.q.out @@ -0,0 +1,60 @@ +PREHOOK: query: create temporary table acid_dtt(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_dtt +POSTHOOK: query: create temporary table acid_dtt(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_dtt +PREHOOK: query: insert into table acid_dtt select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_dtt +POSTHOOK: query: insert into table acid_dtt select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_dtt +POSTHOOK: Lineage: acid_dtt.a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_dtt.b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: select * from acid_dtt order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dtt +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_dtt order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dtt +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa +-1073051226 A34p7oRr2WvUJNf +-1072910839 0iqrc5 +-1072081801 dPkN74F7 +-1072076362 2uLyD28144vklju213J1mr +-1071480828 aw724t8c5558x2xneC624 +-1071363017 Anj0oF +-1070883071 0ruyd6Y50JpdGRf6HqD +-1070551679 iUR3Q +-1069736047 k17Am8uPHWk02cEf1jet +PREHOOK: query: delete from acid_dtt where b = '0ruyd6Y50JpdGRf6HqD' or b = '2uLyD28144vklju213J1mr' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dtt +PREHOOK: Output: default@acid_dtt +POSTHOOK: query: delete from acid_dtt where b = '0ruyd6Y50JpdGRf6HqD' or b = '2uLyD28144vklju213J1mr' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dtt +POSTHOOK: Output: default@acid_dtt +PREHOOK: query: select a,b from acid_dtt order by b +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dtt +#### A masked pattern was here #### +POSTHOOK: query: select a,b from acid_dtt order by b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dtt +#### A masked pattern was here #### +-1072910839 0iqrc5 +-1073051226 A34p7oRr2WvUJNf +-1071363017 Anj0oF +-1071480828 aw724t8c5558x2xneC624 +-1072081801 dPkN74F7 +-1070551679 iUR3Q +-1069736047 k17Am8uPHWk02cEf1jet +-1073279343 oj1YrV5Wa diff --git ql/src/test/results/clientpositive/delete_where_no_match.q.out ql/src/test/results/clientpositive/delete_where_no_match.q.out new file mode 100644 index 0000000..1450ee6 --- /dev/null +++ ql/src/test/results/clientpositive/delete_where_no_match.q.out @@ -0,0 +1,62 @@ +PREHOOK: query: create table acid_dwnm(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_dwnm +POSTHOOK: query: create table acid_dwnm(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_dwnm +PREHOOK: query: insert into table acid_dwnm select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_dwnm +POSTHOOK: query: insert into table acid_dwnm select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_dwnm +POSTHOOK: Lineage: acid_dwnm.a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_dwnm.b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: select * from acid_dwnm order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dwnm +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_dwnm order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dwnm +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa +-1073051226 A34p7oRr2WvUJNf +-1072910839 0iqrc5 +-1072081801 dPkN74F7 +-1072076362 2uLyD28144vklju213J1mr +-1071480828 aw724t8c5558x2xneC624 +-1071363017 Anj0oF +-1070883071 0ruyd6Y50JpdGRf6HqD +-1070551679 iUR3Q +-1069736047 k17Am8uPHWk02cEf1jet +PREHOOK: query: delete from acid_dwnm where b = 'nosuchvalue' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dwnm +PREHOOK: Output: default@acid_dwnm +POSTHOOK: query: delete from acid_dwnm where b = 'nosuchvalue' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dwnm +POSTHOOK: Output: default@acid_dwnm +PREHOOK: query: select a,b from acid_dwnm order by b +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dwnm +#### A masked pattern was here #### +POSTHOOK: query: select a,b from acid_dwnm order by b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dwnm +#### A masked pattern was here #### +-1072910839 0iqrc5 +-1070883071 0ruyd6Y50JpdGRf6HqD +-1072076362 2uLyD28144vklju213J1mr +-1073051226 A34p7oRr2WvUJNf +-1071363017 Anj0oF +-1071480828 aw724t8c5558x2xneC624 +-1072081801 dPkN74F7 +-1070551679 iUR3Q +-1069736047 k17Am8uPHWk02cEf1jet +-1073279343 oj1YrV5Wa diff --git ql/src/test/results/clientpositive/delete_where_non_partitioned.q.out ql/src/test/results/clientpositive/delete_where_non_partitioned.q.out new file mode 100644 index 0000000..d465e8e --- /dev/null +++ ql/src/test/results/clientpositive/delete_where_non_partitioned.q.out @@ -0,0 +1,61 @@ +PREHOOK: query: create table acid_dwnp(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_dwnp +POSTHOOK: query: create table acid_dwnp(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_dwnp +PREHOOK: query: insert into table acid_dwnp select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_dwnp +POSTHOOK: query: insert into table acid_dwnp select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_dwnp +POSTHOOK: Lineage: acid_dwnp.a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_dwnp.b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: select * from acid_dwnp order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dwnp +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_dwnp order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dwnp +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa +-1073051226 A34p7oRr2WvUJNf +-1072910839 0iqrc5 +-1072081801 dPkN74F7 +-1072076362 2uLyD28144vklju213J1mr +-1071480828 aw724t8c5558x2xneC624 +-1071363017 Anj0oF +-1070883071 0ruyd6Y50JpdGRf6HqD +-1070551679 iUR3Q +-1069736047 k17Am8uPHWk02cEf1jet +PREHOOK: query: delete from acid_dwnp where b = '0ruyd6Y50JpdGRf6HqD' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dwnp +PREHOOK: Output: default@acid_dwnp +POSTHOOK: query: delete from acid_dwnp where b = '0ruyd6Y50JpdGRf6HqD' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dwnp +POSTHOOK: Output: default@acid_dwnp +PREHOOK: query: select a,b from acid_dwnp order by b +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dwnp +#### A masked pattern was here #### +POSTHOOK: query: select a,b from acid_dwnp order by b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dwnp +#### A masked pattern was here #### +-1072910839 0iqrc5 +-1072076362 2uLyD28144vklju213J1mr +-1073051226 A34p7oRr2WvUJNf +-1071363017 Anj0oF +-1071480828 aw724t8c5558x2xneC624 +-1072081801 dPkN74F7 +-1070551679 iUR3Q +-1069736047 k17Am8uPHWk02cEf1jet +-1073279343 oj1YrV5Wa diff --git ql/src/test/results/clientpositive/delete_where_partitioned.q.out ql/src/test/results/clientpositive/delete_where_partitioned.q.out new file mode 100644 index 0000000..9f8581b --- /dev/null +++ ql/src/test/results/clientpositive/delete_where_partitioned.q.out @@ -0,0 +1,105 @@ +PREHOOK: query: create table acid_dwp(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_dwp +POSTHOOK: query: create table acid_dwp(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_dwp +PREHOOK: query: insert into table acid_dwp partition (ds='today') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint < 0 order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_dwp@ds=today +POSTHOOK: query: insert into table acid_dwp partition (ds='today') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint < 0 order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_dwp@ds=today +POSTHOOK: Lineage: acid_dwp PARTITION(ds=today).a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_dwp PARTITION(ds=today).b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: insert into table acid_dwp partition (ds='tomorrow') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint > -10000000 order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_dwp@ds=tomorrow +POSTHOOK: query: insert into table acid_dwp partition (ds='tomorrow') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint > -10000000 order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_dwp@ds=tomorrow +POSTHOOK: Lineage: acid_dwp PARTITION(ds=tomorrow).a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_dwp PARTITION(ds=tomorrow).b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: select a,b,ds from acid_dwp order by a, ds +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dwp +PREHOOK: Input: default@acid_dwp@ds=today +PREHOOK: Input: default@acid_dwp@ds=tomorrow +#### A masked pattern was here #### +POSTHOOK: query: select a,b,ds from acid_dwp order by a, ds +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dwp +POSTHOOK: Input: default@acid_dwp@ds=today +POSTHOOK: Input: default@acid_dwp@ds=tomorrow +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa today +-1073051226 A34p7oRr2WvUJNf today +-1072910839 0iqrc5 today +-1072081801 dPkN74F7 today +-1072076362 2uLyD28144vklju213J1mr today +-1071480828 aw724t8c5558x2xneC624 today +-1071363017 Anj0oF today +-1070883071 0ruyd6Y50JpdGRf6HqD today +-1070551679 iUR3Q today +-1069736047 k17Am8uPHWk02cEf1jet today +-9676535 MmMPCF2 tomorrow +-9462165 7WLVW6F4h71Dgk7 tomorrow +-9329892 e7sC5M0H5K6EgSTf41X tomorrow +-9175632 UUBET8444iJDvjUlq3en tomorrow +-9011819 A6CX2HDWN8 tomorrow +-8987676 FhXANp2KDtMmA2gFd778pA tomorrow +-8413710 81Rg5rR0IaInWw tomorrow +-8230445 K6J1LIb5 tomorrow +-7980033 HtI02nss6t8S0fqH4vcLkCD tomorrow +-6882225 r6gCtT4Tgo5rG tomorrow +PREHOOK: query: delete from acid_dwp where a = '-1071363017' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dwp +PREHOOK: Input: default@acid_dwp@ds=today +PREHOOK: Input: default@acid_dwp@ds=tomorrow +PREHOOK: Output: default@acid_dwp@ds=today +PREHOOK: Output: default@acid_dwp@ds=tomorrow +POSTHOOK: query: delete from acid_dwp where a = '-1071363017' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dwp +POSTHOOK: Input: default@acid_dwp@ds=today +POSTHOOK: Input: default@acid_dwp@ds=tomorrow +POSTHOOK: Output: default@acid_dwp@ds=today +POSTHOOK: Output: default@acid_dwp@ds=tomorrow +PREHOOK: query: select * from acid_dwp order by a, ds +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dwp +PREHOOK: Input: default@acid_dwp@ds=today +PREHOOK: Input: default@acid_dwp@ds=tomorrow +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_dwp order by a, ds +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dwp +POSTHOOK: Input: default@acid_dwp@ds=today +POSTHOOK: Input: default@acid_dwp@ds=tomorrow +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa today +-1073051226 A34p7oRr2WvUJNf today +-1072910839 0iqrc5 today +-1072081801 dPkN74F7 today +-1072076362 2uLyD28144vklju213J1mr today +-1071480828 aw724t8c5558x2xneC624 today +-1070883071 0ruyd6Y50JpdGRf6HqD today +-1070551679 iUR3Q today +-1069736047 k17Am8uPHWk02cEf1jet today +-9676535 MmMPCF2 tomorrow +-9462165 7WLVW6F4h71Dgk7 tomorrow +-9329892 e7sC5M0H5K6EgSTf41X tomorrow +-9175632 UUBET8444iJDvjUlq3en tomorrow +-9011819 A6CX2HDWN8 tomorrow +-8987676 FhXANp2KDtMmA2gFd778pA tomorrow +-8413710 81Rg5rR0IaInWw tomorrow +-8230445 K6J1LIb5 tomorrow +-7980033 HtI02nss6t8S0fqH4vcLkCD tomorrow +-6882225 r6gCtT4Tgo5rG tomorrow diff --git ql/src/test/results/clientpositive/delete_whole_partition.q.out ql/src/test/results/clientpositive/delete_whole_partition.q.out new file mode 100644 index 0000000..a2408eb --- /dev/null +++ ql/src/test/results/clientpositive/delete_whole_partition.q.out @@ -0,0 +1,92 @@ +PREHOOK: query: create table acid_dwhp(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_dwhp +POSTHOOK: query: create table acid_dwhp(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_dwhp +PREHOOK: query: insert into table acid_dwhp partition (ds='today') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint < 0 order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_dwhp@ds=today +POSTHOOK: query: insert into table acid_dwhp partition (ds='today') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint < 0 order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_dwhp@ds=today +POSTHOOK: Lineage: acid_dwhp PARTITION(ds=today).a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_dwhp PARTITION(ds=today).b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: insert into table acid_dwhp partition (ds='tomorrow') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint > -10000000 order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_dwhp@ds=tomorrow +POSTHOOK: query: insert into table acid_dwhp partition (ds='tomorrow') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint > -10000000 order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_dwhp@ds=tomorrow +POSTHOOK: Lineage: acid_dwhp PARTITION(ds=tomorrow).a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_dwhp PARTITION(ds=tomorrow).b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: select a,b,ds from acid_dwhp order by a, ds +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dwhp +PREHOOK: Input: default@acid_dwhp@ds=today +PREHOOK: Input: default@acid_dwhp@ds=tomorrow +#### A masked pattern was here #### +POSTHOOK: query: select a,b,ds from acid_dwhp order by a, ds +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dwhp +POSTHOOK: Input: default@acid_dwhp@ds=today +POSTHOOK: Input: default@acid_dwhp@ds=tomorrow +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa today +-1073051226 A34p7oRr2WvUJNf today +-1072910839 0iqrc5 today +-1072081801 dPkN74F7 today +-1072076362 2uLyD28144vklju213J1mr today +-1071480828 aw724t8c5558x2xneC624 today +-1071363017 Anj0oF today +-1070883071 0ruyd6Y50JpdGRf6HqD today +-1070551679 iUR3Q today +-1069736047 k17Am8uPHWk02cEf1jet today +-9676535 MmMPCF2 tomorrow +-9462165 7WLVW6F4h71Dgk7 tomorrow +-9329892 e7sC5M0H5K6EgSTf41X tomorrow +-9175632 UUBET8444iJDvjUlq3en tomorrow +-9011819 A6CX2HDWN8 tomorrow +-8987676 FhXANp2KDtMmA2gFd778pA tomorrow +-8413710 81Rg5rR0IaInWw tomorrow +-8230445 K6J1LIb5 tomorrow +-7980033 HtI02nss6t8S0fqH4vcLkCD tomorrow +-6882225 r6gCtT4Tgo5rG tomorrow +PREHOOK: query: delete from acid_dwhp where ds = 'today' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dwhp +PREHOOK: Input: default@acid_dwhp@ds=today +PREHOOK: Output: default@acid_dwhp@ds=today +POSTHOOK: query: delete from acid_dwhp where ds = 'today' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dwhp +POSTHOOK: Input: default@acid_dwhp@ds=today +POSTHOOK: Output: default@acid_dwhp@ds=today +PREHOOK: query: select * from acid_dwhp order by a, ds +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dwhp +PREHOOK: Input: default@acid_dwhp@ds=today +PREHOOK: Input: default@acid_dwhp@ds=tomorrow +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_dwhp order by a, ds +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dwhp +POSTHOOK: Input: default@acid_dwhp@ds=today +POSTHOOK: Input: default@acid_dwhp@ds=tomorrow +#### A masked pattern was here #### +-9676535 MmMPCF2 tomorrow +-9462165 7WLVW6F4h71Dgk7 tomorrow +-9329892 e7sC5M0H5K6EgSTf41X tomorrow +-9175632 UUBET8444iJDvjUlq3en tomorrow +-9011819 A6CX2HDWN8 tomorrow +-8987676 FhXANp2KDtMmA2gFd778pA tomorrow +-8413710 81Rg5rR0IaInWw tomorrow +-8230445 K6J1LIb5 tomorrow +-7980033 HtI02nss6t8S0fqH4vcLkCD tomorrow +-6882225 r6gCtT4Tgo5rG tomorrow diff --git ql/src/test/results/clientpositive/insert_orig_table.q.out ql/src/test/results/clientpositive/insert_orig_table.q.out new file mode 100644 index 0000000..97a284b --- /dev/null +++ ql/src/test/results/clientpositive/insert_orig_table.q.out @@ -0,0 +1,80 @@ +PREHOOK: query: create table acid_iot( + ctinyint TINYINT, + csmallint SMALLINT, + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE, + cstring1 STRING, + cstring2 STRING, + ctimestamp1 TIMESTAMP, + ctimestamp2 TIMESTAMP, + cboolean1 BOOLEAN, + cboolean2 BOOLEAN) clustered by (cint) into 1 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_iot +POSTHOOK: query: create table acid_iot( + ctinyint TINYINT, + csmallint SMALLINT, + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE, + cstring1 STRING, + cstring2 STRING, + ctimestamp1 TIMESTAMP, + ctimestamp2 TIMESTAMP, + cboolean1 BOOLEAN, + cboolean2 BOOLEAN) clustered by (cint) into 1 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_iot +PREHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/alltypesorc" into table acid_iot +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@acid_iot +POSTHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/alltypesorc" into table acid_iot +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@acid_iot +PREHOOK: query: select count(*) from acid_iot +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_iot +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid_iot +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_iot +#### A masked pattern was here #### +12288 +PREHOOK: query: insert into table acid_iot select ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, + cboolean1, cboolean2 from alltypesorc where cint < 0 order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_iot +POSTHOOK: query: insert into table acid_iot select ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, + cboolean1, cboolean2 from alltypesorc where cint < 0 order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_iot +POSTHOOK: Lineage: acid_iot.cbigint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ] +POSTHOOK: Lineage: acid_iot.cboolean1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean1, type:boolean, comment:null), ] +POSTHOOK: Lineage: acid_iot.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] +POSTHOOK: Lineage: acid_iot.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: acid_iot.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: acid_iot.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_iot.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] +POSTHOOK: Lineage: acid_iot.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: acid_iot.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] +POSTHOOK: Lineage: acid_iot.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] +POSTHOOK: Lineage: acid_iot.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] +POSTHOOK: Lineage: acid_iot.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +PREHOOK: query: select count(*) from acid_iot +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_iot +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid_iot +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_iot +#### A masked pattern was here #### +12298 diff --git ql/src/test/results/clientpositive/insert_update_delete.q.out ql/src/test/results/clientpositive/insert_update_delete.q.out new file mode 100644 index 0000000..e9f9984 --- /dev/null +++ ql/src/test/results/clientpositive/insert_update_delete.q.out @@ -0,0 +1,78 @@ +PREHOOK: query: create table acid_iud(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_iud +POSTHOOK: query: create table acid_iud(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_iud +PREHOOK: query: insert into table acid_iud select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint < 0 order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_iud +POSTHOOK: query: insert into table acid_iud select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint < 0 order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_iud +POSTHOOK: Lineage: acid_iud.a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_iud.b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: select a,b from acid_iud order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_iud +#### A masked pattern was here #### +POSTHOOK: query: select a,b from acid_iud order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_iud +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa +-1073051226 A34p7oRr2WvUJNf +-1072910839 0iqrc5 +-1072081801 dPkN74F7 +-1072076362 2uLyD28144vklju213J1mr +-1071480828 aw724t8c5558x2xneC624 +-1071363017 Anj0oF +-1070883071 0ruyd6Y50JpdGRf6HqD +-1070551679 iUR3Q +-1069736047 k17Am8uPHWk02cEf1jet +PREHOOK: query: update acid_iud set b = 'fred' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_iud +PREHOOK: Output: default@acid_iud +POSTHOOK: query: update acid_iud set b = 'fred' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_iud +POSTHOOK: Output: default@acid_iud +PREHOOK: query: select a,b from acid_iud order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_iud +#### A masked pattern was here #### +POSTHOOK: query: select a,b from acid_iud order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_iud +#### A masked pattern was here #### +-1073279343 fred +-1073051226 fred +-1072910839 fred +-1072081801 fred +-1072076362 fred +-1071480828 fred +-1071363017 fred +-1070883071 fred +-1070551679 fred +-1069736047 fred +PREHOOK: query: delete from acid_iud +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_iud +PREHOOK: Output: default@acid_iud +POSTHOOK: query: delete from acid_iud +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_iud +POSTHOOK: Output: default@acid_iud +PREHOOK: query: select a,b from acid_iud order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_iud +#### A masked pattern was here #### +POSTHOOK: query: select a,b from acid_iud order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_iud +#### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/insert_values_dynamic_partitioned.q.out ql/src/test/results/clientpositive/insert_values_dynamic_partitioned.q.out new file mode 100644 index 0000000..daea059 --- /dev/null +++ ql/src/test/results/clientpositive/insert_values_dynamic_partitioned.q.out @@ -0,0 +1,45 @@ +PREHOOK: query: create table ivdp(i int, + de decimal(5,2), + vc varchar(128)) partitioned by (ds string) clustered by (i) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ivdp +POSTHOOK: query: create table ivdp(i int, + de decimal(5,2), + vc varchar(128)) partitioned by (ds string) clustered by (i) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ivdp +PREHOOK: query: insert into table ivdp partition (ds) values + (1, 109.23, 'and everywhere that mary went', 'today'), + (6553, 923.19, 'the lamb was sure to go', 'tomorrow') +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@ivdp +POSTHOOK: query: insert into table ivdp partition (ds) values + (1, 109.23, 'and everywhere that mary went', 'today'), + (6553, 923.19, 'the lamb was sure to go', 'tomorrow') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@ivdp@ds=today +POSTHOOK: Output: default@ivdp@ds=tomorrow +POSTHOOK: Lineage: ivdp PARTITION(ds=today).de EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: ivdp PARTITION(ds=today).i EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: ivdp PARTITION(ds=today).vc EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: ivdp PARTITION(ds=tomorrow).de EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: ivdp PARTITION(ds=tomorrow).i EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: ivdp PARTITION(ds=tomorrow).vc EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +PREHOOK: query: select * from ivdp order by ds +PREHOOK: type: QUERY +PREHOOK: Input: default@ivdp +PREHOOK: Input: default@ivdp@ds=today +PREHOOK: Input: default@ivdp@ds=tomorrow +#### A masked pattern was here #### +POSTHOOK: query: select * from ivdp order by ds +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ivdp +POSTHOOK: Input: default@ivdp@ds=today +POSTHOOK: Input: default@ivdp@ds=tomorrow +#### A masked pattern was here #### +1 109.23 and everywhere that mary went today +6553 923.19 the lamb was sure to go tomorrow diff --git ql/src/test/results/clientpositive/insert_values_non_partitioned.q.out ql/src/test/results/clientpositive/insert_values_non_partitioned.q.out new file mode 100644 index 0000000..a6f8ac4 --- /dev/null +++ ql/src/test/results/clientpositive/insert_values_non_partitioned.q.out @@ -0,0 +1,64 @@ +PREHOOK: query: create table acid_ivnp(ti tinyint, + si smallint, + i int, + bi bigint, + f float, + d double, + de decimal(5,2), + t timestamp, + dt date, + s string, + vc varchar(128), + ch char(12)) clustered by (i) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_ivnp +POSTHOOK: query: create table acid_ivnp(ti tinyint, + si smallint, + i int, + bi bigint, + f float, + d double, + de decimal(5,2), + t timestamp, + dt date, + s string, + vc varchar(128), + ch char(12)) clustered by (i) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_ivnp +PREHOOK: query: insert into table acid_ivnp values + (1, 257, 65537, 4294967297, 3.14, 3.141592654, 109.23, '2014-08-25 17:21:30.0', '2014-08-25', 'mary had a little lamb', 'ring around the rosie', 'red'), + (3, 25, 6553, 429496729, 0.14, 1923.141592654, 1.2301, '2014-08-24 17:21:30.0', '2014-08-26', 'its fleece was white as snow', 'a pocket full of posies', 'blue' ) +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@acid_ivnp +POSTHOOK: query: insert into table acid_ivnp values + (1, 257, 65537, 4294967297, 3.14, 3.141592654, 109.23, '2014-08-25 17:21:30.0', '2014-08-25', 'mary had a little lamb', 'ring around the rosie', 'red'), + (3, 25, 6553, 429496729, 0.14, 1923.141592654, 1.2301, '2014-08-24 17:21:30.0', '2014-08-26', 'its fleece was white as snow', 'a pocket full of posies', 'blue' ) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@acid_ivnp +POSTHOOK: Lineage: acid_ivnp.bi EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col4, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivnp.ch EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col12, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivnp.d EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col6, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivnp.de EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col7, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivnp.dt EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col9, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivnp.f EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col5, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivnp.i EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivnp.s SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col10, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivnp.si EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivnp.t EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col8, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivnp.ti EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivnp.vc EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col11, type:string, comment:), ] +PREHOOK: query: select ti, si, i, bi, f, d, de, t, dt, s, vc, ch from acid_ivnp order by ti +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_ivnp +#### A masked pattern was here #### +POSTHOOK: query: select ti, si, i, bi, f, d, de, t, dt, s, vc, ch from acid_ivnp order by ti +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_ivnp +#### A masked pattern was here #### +1 257 65537 4294967297 3.14 3.141592654 109.23 2014-08-25 17:21:30 2014-08-25 mary had a little lamb ring around the rosie red +3 25 6553 429496729 0.14 1923.141592654 1.23 2014-08-24 17:21:30 2014-08-26 its fleece was white as snow a pocket full of posies blue diff --git ql/src/test/results/clientpositive/insert_values_orig_table.q.out ql/src/test/results/clientpositive/insert_values_orig_table.q.out new file mode 100644 index 0000000..69220ec --- /dev/null +++ ql/src/test/results/clientpositive/insert_values_orig_table.q.out @@ -0,0 +1,82 @@ +PREHOOK: query: create table acid_ivot( + ctinyint TINYINT, + csmallint SMALLINT, + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE, + cstring1 STRING, + cstring2 STRING, + ctimestamp1 TIMESTAMP, + ctimestamp2 TIMESTAMP, + cboolean1 BOOLEAN, + cboolean2 BOOLEAN) clustered by (cint) into 1 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_ivot +POSTHOOK: query: create table acid_ivot( + ctinyint TINYINT, + csmallint SMALLINT, + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE, + cstring1 STRING, + cstring2 STRING, + ctimestamp1 TIMESTAMP, + ctimestamp2 TIMESTAMP, + cboolean1 BOOLEAN, + cboolean2 BOOLEAN) clustered by (cint) into 1 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_ivot +PREHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/alltypesorc" into table acid_ivot +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@acid_ivot +POSTHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/alltypesorc" into table acid_ivot +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@acid_ivot +PREHOOK: query: select count(*) from acid_ivot +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_ivot +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid_ivot +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_ivot +#### A masked pattern was here #### +12288 +PREHOOK: query: insert into table acid_ivot values + (1, 2, 3, 4, 3.14, 2.34, 'fred', 'bob', '2014-09-01 10:34:23.111', '1944-06-06 06:00:00', true, true), + (111, 222, 3333, 444, 13.14, 10239302.34239320, 'fred', 'bob', '2014-09-01 10:34:23.111', '1944-06-06 06:00:00', true, true) +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@acid_ivot +POSTHOOK: query: insert into table acid_ivot values + (1, 2, 3, 4, 3.14, 2.34, 'fred', 'bob', '2014-09-01 10:34:23.111', '1944-06-06 06:00:00', true, true), + (111, 222, 3333, 444, 13.14, 10239302.34239320, 'fred', 'bob', '2014-09-01 10:34:23.111', '1944-06-06 06:00:00', true, true) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@acid_ivot +POSTHOOK: Lineage: acid_ivot.cbigint EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col4, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.cboolean1 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col11, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.cboolean2 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col12, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.cdouble EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col6, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.cfloat EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col5, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.cint EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.csmallint EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.cstring1 SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col7, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.cstring2 SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col8, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.ctimestamp1 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col9, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.ctimestamp2 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col10, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.ctinyint EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: select count(*) from acid_ivot +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_ivot +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid_ivot +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_ivot +#### A masked pattern was here #### +12290 diff --git ql/src/test/results/clientpositive/insert_values_partitioned.q.out ql/src/test/results/clientpositive/insert_values_partitioned.q.out new file mode 100644 index 0000000..9fb89ff --- /dev/null +++ ql/src/test/results/clientpositive/insert_values_partitioned.q.out @@ -0,0 +1,66 @@ +PREHOOK: query: create table acid_ivp(ti tinyint, + si smallint, + i int, + bi bigint, + f float, + d double, + de decimal(5,2), + t timestamp, + dt date, + s string, + vc varchar(128), + ch char(12)) partitioned by (ds string) clustered by (i) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_ivp +POSTHOOK: query: create table acid_ivp(ti tinyint, + si smallint, + i int, + bi bigint, + f float, + d double, + de decimal(5,2), + t timestamp, + dt date, + s string, + vc varchar(128), + ch char(12)) partitioned by (ds string) clustered by (i) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_ivp +PREHOOK: query: insert into table acid_ivp partition (ds='today') values + (1, 257, 65537, 4294967297, 3.14, 3.141592654, 109.23, '2014-08-25 17:21:30.0', '2014-08-25', 'mary had a little lamb', 'ring around the rosie', 'red'), + (3, 25, 6553, 429496729, 0.14, 1923.141592654, 1.2301, '2014-08-24 17:21:30.0', '2014-08-26', 'its fleece was white as snow', 'a pocket full of posies', 'blue') +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@acid_ivp@ds=today +POSTHOOK: query: insert into table acid_ivp partition (ds='today') values + (1, 257, 65537, 4294967297, 3.14, 3.141592654, 109.23, '2014-08-25 17:21:30.0', '2014-08-25', 'mary had a little lamb', 'ring around the rosie', 'red'), + (3, 25, 6553, 429496729, 0.14, 1923.141592654, 1.2301, '2014-08-24 17:21:30.0', '2014-08-26', 'its fleece was white as snow', 'a pocket full of posies', 'blue') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@acid_ivp@ds=today +POSTHOOK: Lineage: acid_ivp PARTITION(ds=today).bi EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col4, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivp PARTITION(ds=today).ch EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col12, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivp PARTITION(ds=today).d EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col6, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivp PARTITION(ds=today).de EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col7, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivp PARTITION(ds=today).dt EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col9, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivp PARTITION(ds=today).f EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col5, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivp PARTITION(ds=today).i EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivp PARTITION(ds=today).s SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col10, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivp PARTITION(ds=today).si EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivp PARTITION(ds=today).t EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col8, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivp PARTITION(ds=today).ti EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivp PARTITION(ds=today).vc EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col11, type:string, comment:), ] +PREHOOK: query: select * from acid_ivp order by i +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_ivp +PREHOOK: Input: default@acid_ivp@ds=today +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_ivp order by i +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_ivp +POSTHOOK: Input: default@acid_ivp@ds=today +#### A masked pattern was here #### +3 25 6553 429496729 0.14 1923.141592654 1.23 2014-08-24 17:21:30 2014-08-26 its fleece was white as snow a pocket full of posies blue today +1 257 65537 4294967297 3.14 3.141592654 109.23 2014-08-25 17:21:30 2014-08-25 mary had a little lamb ring around the rosie red today diff --git ql/src/test/results/clientpositive/insert_values_tmp_table.q.out ql/src/test/results/clientpositive/insert_values_tmp_table.q.out new file mode 100644 index 0000000..95d6372 --- /dev/null +++ ql/src/test/results/clientpositive/insert_values_tmp_table.q.out @@ -0,0 +1,33 @@ +PREHOOK: query: create temporary table acid_ivtt(i int, de decimal(5,2), vc varchar(128)) clustered by (vc) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_ivtt +POSTHOOK: query: create temporary table acid_ivtt(i int, de decimal(5,2), vc varchar(128)) clustered by (vc) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_ivtt +PREHOOK: query: insert into table acid_ivtt values + (1, 109.23, 'mary had a little lamb'), + (429496729, 0.14, 'its fleece was white as snow') +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@acid_ivtt +POSTHOOK: query: insert into table acid_ivtt values + (1, 109.23, 'mary had a little lamb'), + (429496729, 0.14, 'its fleece was white as snow') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@acid_ivtt +POSTHOOK: Lineage: acid_ivtt.de EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivtt.i EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivtt.vc EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +PREHOOK: query: select i, de, vc from acid_ivtt order by i +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_ivtt +#### A masked pattern was here #### +POSTHOOK: query: select i, de, vc from acid_ivtt order by i +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_ivtt +#### A masked pattern was here #### +1 109.23 mary had a little lamb +429496729 0.14 its fleece was white as snow diff --git ql/src/test/results/clientpositive/tez/delete_all_non_partitioned.q.out ql/src/test/results/clientpositive/tez/delete_all_non_partitioned.q.out new file mode 100644 index 0000000..0d428ca --- /dev/null +++ ql/src/test/results/clientpositive/tez/delete_all_non_partitioned.q.out @@ -0,0 +1,52 @@ +PREHOOK: query: create table acid_danp(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_danp +POSTHOOK: query: create table acid_danp(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_danp +PREHOOK: query: insert into table acid_danp select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint < 0 order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_danp +POSTHOOK: query: insert into table acid_danp select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint < 0 order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_danp +POSTHOOK: Lineage: acid_danp.a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_danp.b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: select a,b from acid_danp order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_danp +#### A masked pattern was here #### +POSTHOOK: query: select a,b from acid_danp order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_danp +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa +-1073051226 A34p7oRr2WvUJNf +-1072910839 0iqrc5 +-1072081801 dPkN74F7 +-1072076362 2uLyD28144vklju213J1mr +-1071480828 aw724t8c5558x2xneC624 +-1071363017 Anj0oF +-1070883071 0ruyd6Y50JpdGRf6HqD +-1070551679 iUR3Q +-1069736047 k17Am8uPHWk02cEf1jet +PREHOOK: query: delete from acid_danp +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_danp +PREHOOK: Output: default@acid_danp +POSTHOOK: query: delete from acid_danp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_danp +POSTHOOK: Output: default@acid_danp +PREHOOK: query: select a,b from acid_danp +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_danp +#### A masked pattern was here #### +POSTHOOK: query: select a,b from acid_danp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_danp +#### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/tez/delete_all_partitioned.q.out ql/src/test/results/clientpositive/tez/delete_all_partitioned.q.out new file mode 100644 index 0000000..4486323 --- /dev/null +++ ql/src/test/results/clientpositive/tez/delete_all_partitioned.q.out @@ -0,0 +1,86 @@ +PREHOOK: query: create table acid_dap(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_dap +POSTHOOK: query: create table acid_dap(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_dap +PREHOOK: query: insert into table acid_dap partition (ds='today') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint < 0 order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_dap@ds=today +POSTHOOK: query: insert into table acid_dap partition (ds='today') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint < 0 order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_dap@ds=today +POSTHOOK: Lineage: acid_dap PARTITION(ds=today).a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_dap PARTITION(ds=today).b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: insert into table acid_dap partition (ds='tomorrow') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint > 1000 order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_dap@ds=tomorrow +POSTHOOK: query: insert into table acid_dap partition (ds='tomorrow') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint > 1000 order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_dap@ds=tomorrow +POSTHOOK: Lineage: acid_dap PARTITION(ds=tomorrow).a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_dap PARTITION(ds=tomorrow).b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: select a,b,ds from acid_dap order by a,b +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dap +PREHOOK: Input: default@acid_dap@ds=today +PREHOOK: Input: default@acid_dap@ds=tomorrow +#### A masked pattern was here #### +POSTHOOK: query: select a,b,ds from acid_dap order by a,b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dap +POSTHOOK: Input: default@acid_dap@ds=today +POSTHOOK: Input: default@acid_dap@ds=tomorrow +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa today +-1073051226 A34p7oRr2WvUJNf today +-1072910839 0iqrc5 today +-1072081801 dPkN74F7 today +-1072076362 2uLyD28144vklju213J1mr today +-1071480828 aw724t8c5558x2xneC624 today +-1071363017 Anj0oF today +-1070883071 0ruyd6Y50JpdGRf6HqD today +-1070551679 iUR3Q today +-1069736047 k17Am8uPHWk02cEf1jet today +6981 NULL tomorrow +6981 1FNNhmiFLGw425NA13g tomorrow +6981 4KhrrQ0nJ7bMNTvhSCA tomorrow +6981 K630vaVf tomorrow +6981 Y5x3JuI3M8jngv5N tomorrow +6981 YdG61y00526u5 tomorrow +6981 a3EhVU6Wuy7ycJ7wY7h2gv tomorrow +6981 o4lvY20511w0EOX3P3I82p63 tomorrow +6981 o5mb0QP5Y48Qd4vdB0 tomorrow +6981 sF2CRfgt2K tomorrow +PREHOOK: query: delete from acid_dap +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dap +PREHOOK: Input: default@acid_dap@ds=today +PREHOOK: Input: default@acid_dap@ds=tomorrow +PREHOOK: Output: default@acid_dap@ds=today +PREHOOK: Output: default@acid_dap@ds=tomorrow +POSTHOOK: query: delete from acid_dap +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dap +POSTHOOK: Input: default@acid_dap@ds=today +POSTHOOK: Input: default@acid_dap@ds=tomorrow +POSTHOOK: Output: default@acid_dap@ds=today +POSTHOOK: Output: default@acid_dap@ds=tomorrow +PREHOOK: query: select * from acid_dap +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dap +PREHOOK: Input: default@acid_dap@ds=today +PREHOOK: Input: default@acid_dap@ds=tomorrow +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_dap +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dap +POSTHOOK: Input: default@acid_dap@ds=today +POSTHOOK: Input: default@acid_dap@ds=tomorrow +#### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/tez/delete_orig_table.q.out ql/src/test/results/clientpositive/tez/delete_orig_table.q.out new file mode 100644 index 0000000..a036b06 --- /dev/null +++ ql/src/test/results/clientpositive/tez/delete_orig_table.q.out @@ -0,0 +1,61 @@ +PREHOOK: query: create table acid_dot( + ctinyint TINYINT, + csmallint SMALLINT, + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE, + cstring1 STRING, + cstring2 STRING, + ctimestamp1 TIMESTAMP, + ctimestamp2 TIMESTAMP, + cboolean1 BOOLEAN, +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_dot +POSTHOOK: query: create table acid_dot( + ctinyint TINYINT, + csmallint SMALLINT, + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE, + cstring1 STRING, + cstring2 STRING, + ctimestamp1 TIMESTAMP, + ctimestamp2 TIMESTAMP, + cboolean1 BOOLEAN, +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_dot +PREHOOK: query: select count(*) from acid_dot +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dot +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid_dot +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dot +#### A masked pattern was here #### +12288 +PREHOOK: query: delete from acid_dot where cint < -1070551679 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dot +PREHOOK: Output: default@acid_dot +POSTHOOK: query: delete from acid_dot where cint < -1070551679 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dot +POSTHOOK: Output: default@acid_dot +PREHOOK: query: select count(*) from acid_dot +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dot +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid_dot +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dot +#### A masked pattern was here #### +12280 +#### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/tez/delete_tmp_table.q.out ql/src/test/results/clientpositive/tez/delete_tmp_table.q.out new file mode 100644 index 0000000..ca568b3 --- /dev/null +++ ql/src/test/results/clientpositive/tez/delete_tmp_table.q.out @@ -0,0 +1,60 @@ +PREHOOK: query: create temporary table acid_dtt(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_dtt +POSTHOOK: query: create temporary table acid_dtt(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_dtt +PREHOOK: query: insert into table acid_dtt select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_dtt +POSTHOOK: query: insert into table acid_dtt select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_dtt +POSTHOOK: Lineage: acid_dtt.a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_dtt.b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: select * from acid_dtt order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dtt +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_dtt order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dtt +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa +-1073051226 A34p7oRr2WvUJNf +-1072910839 0iqrc5 +-1072081801 dPkN74F7 +-1072076362 2uLyD28144vklju213J1mr +-1071480828 aw724t8c5558x2xneC624 +-1071363017 Anj0oF +-1070883071 0ruyd6Y50JpdGRf6HqD +-1070551679 iUR3Q +-1069736047 k17Am8uPHWk02cEf1jet +PREHOOK: query: delete from acid_dtt where b = '0ruyd6Y50JpdGRf6HqD' or b = '2uLyD28144vklju213J1mr' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dtt +PREHOOK: Output: default@acid_dtt +POSTHOOK: query: delete from acid_dtt where b = '0ruyd6Y50JpdGRf6HqD' or b = '2uLyD28144vklju213J1mr' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dtt +POSTHOOK: Output: default@acid_dtt +PREHOOK: query: select a,b from acid_dtt order by b +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dtt +#### A masked pattern was here #### +POSTHOOK: query: select a,b from acid_dtt order by b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dtt +#### A masked pattern was here #### +-1072910839 0iqrc5 +-1073051226 A34p7oRr2WvUJNf +-1071363017 Anj0oF +-1071480828 aw724t8c5558x2xneC624 +-1072081801 dPkN74F7 +-1070551679 iUR3Q +-1069736047 k17Am8uPHWk02cEf1jet +-1073279343 oj1YrV5Wa diff --git ql/src/test/results/clientpositive/tez/delete_where_no_match.q.out ql/src/test/results/clientpositive/tez/delete_where_no_match.q.out new file mode 100644 index 0000000..1450ee6 --- /dev/null +++ ql/src/test/results/clientpositive/tez/delete_where_no_match.q.out @@ -0,0 +1,62 @@ +PREHOOK: query: create table acid_dwnm(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_dwnm +POSTHOOK: query: create table acid_dwnm(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_dwnm +PREHOOK: query: insert into table acid_dwnm select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_dwnm +POSTHOOK: query: insert into table acid_dwnm select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_dwnm +POSTHOOK: Lineage: acid_dwnm.a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_dwnm.b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: select * from acid_dwnm order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dwnm +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_dwnm order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dwnm +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa +-1073051226 A34p7oRr2WvUJNf +-1072910839 0iqrc5 +-1072081801 dPkN74F7 +-1072076362 2uLyD28144vklju213J1mr +-1071480828 aw724t8c5558x2xneC624 +-1071363017 Anj0oF +-1070883071 0ruyd6Y50JpdGRf6HqD +-1070551679 iUR3Q +-1069736047 k17Am8uPHWk02cEf1jet +PREHOOK: query: delete from acid_dwnm where b = 'nosuchvalue' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dwnm +PREHOOK: Output: default@acid_dwnm +POSTHOOK: query: delete from acid_dwnm where b = 'nosuchvalue' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dwnm +POSTHOOK: Output: default@acid_dwnm +PREHOOK: query: select a,b from acid_dwnm order by b +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dwnm +#### A masked pattern was here #### +POSTHOOK: query: select a,b from acid_dwnm order by b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dwnm +#### A masked pattern was here #### +-1072910839 0iqrc5 +-1070883071 0ruyd6Y50JpdGRf6HqD +-1072076362 2uLyD28144vklju213J1mr +-1073051226 A34p7oRr2WvUJNf +-1071363017 Anj0oF +-1071480828 aw724t8c5558x2xneC624 +-1072081801 dPkN74F7 +-1070551679 iUR3Q +-1069736047 k17Am8uPHWk02cEf1jet +-1073279343 oj1YrV5Wa diff --git ql/src/test/results/clientpositive/tez/delete_where_non_partitioned.q.out ql/src/test/results/clientpositive/tez/delete_where_non_partitioned.q.out new file mode 100644 index 0000000..d465e8e --- /dev/null +++ ql/src/test/results/clientpositive/tez/delete_where_non_partitioned.q.out @@ -0,0 +1,61 @@ +PREHOOK: query: create table acid_dwnp(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_dwnp +POSTHOOK: query: create table acid_dwnp(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_dwnp +PREHOOK: query: insert into table acid_dwnp select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_dwnp +POSTHOOK: query: insert into table acid_dwnp select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_dwnp +POSTHOOK: Lineage: acid_dwnp.a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_dwnp.b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: select * from acid_dwnp order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dwnp +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_dwnp order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dwnp +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa +-1073051226 A34p7oRr2WvUJNf +-1072910839 0iqrc5 +-1072081801 dPkN74F7 +-1072076362 2uLyD28144vklju213J1mr +-1071480828 aw724t8c5558x2xneC624 +-1071363017 Anj0oF +-1070883071 0ruyd6Y50JpdGRf6HqD +-1070551679 iUR3Q +-1069736047 k17Am8uPHWk02cEf1jet +PREHOOK: query: delete from acid_dwnp where b = '0ruyd6Y50JpdGRf6HqD' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dwnp +PREHOOK: Output: default@acid_dwnp +POSTHOOK: query: delete from acid_dwnp where b = '0ruyd6Y50JpdGRf6HqD' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dwnp +POSTHOOK: Output: default@acid_dwnp +PREHOOK: query: select a,b from acid_dwnp order by b +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dwnp +#### A masked pattern was here #### +POSTHOOK: query: select a,b from acid_dwnp order by b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dwnp +#### A masked pattern was here #### +-1072910839 0iqrc5 +-1072076362 2uLyD28144vklju213J1mr +-1073051226 A34p7oRr2WvUJNf +-1071363017 Anj0oF +-1071480828 aw724t8c5558x2xneC624 +-1072081801 dPkN74F7 +-1070551679 iUR3Q +-1069736047 k17Am8uPHWk02cEf1jet +-1073279343 oj1YrV5Wa diff --git ql/src/test/results/clientpositive/tez/delete_where_partitioned.q.out ql/src/test/results/clientpositive/tez/delete_where_partitioned.q.out new file mode 100644 index 0000000..9f8581b --- /dev/null +++ ql/src/test/results/clientpositive/tez/delete_where_partitioned.q.out @@ -0,0 +1,105 @@ +PREHOOK: query: create table acid_dwp(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_dwp +POSTHOOK: query: create table acid_dwp(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_dwp +PREHOOK: query: insert into table acid_dwp partition (ds='today') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint < 0 order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_dwp@ds=today +POSTHOOK: query: insert into table acid_dwp partition (ds='today') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint < 0 order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_dwp@ds=today +POSTHOOK: Lineage: acid_dwp PARTITION(ds=today).a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_dwp PARTITION(ds=today).b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: insert into table acid_dwp partition (ds='tomorrow') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint > -10000000 order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_dwp@ds=tomorrow +POSTHOOK: query: insert into table acid_dwp partition (ds='tomorrow') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint > -10000000 order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_dwp@ds=tomorrow +POSTHOOK: Lineage: acid_dwp PARTITION(ds=tomorrow).a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_dwp PARTITION(ds=tomorrow).b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: select a,b,ds from acid_dwp order by a, ds +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dwp +PREHOOK: Input: default@acid_dwp@ds=today +PREHOOK: Input: default@acid_dwp@ds=tomorrow +#### A masked pattern was here #### +POSTHOOK: query: select a,b,ds from acid_dwp order by a, ds +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dwp +POSTHOOK: Input: default@acid_dwp@ds=today +POSTHOOK: Input: default@acid_dwp@ds=tomorrow +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa today +-1073051226 A34p7oRr2WvUJNf today +-1072910839 0iqrc5 today +-1072081801 dPkN74F7 today +-1072076362 2uLyD28144vklju213J1mr today +-1071480828 aw724t8c5558x2xneC624 today +-1071363017 Anj0oF today +-1070883071 0ruyd6Y50JpdGRf6HqD today +-1070551679 iUR3Q today +-1069736047 k17Am8uPHWk02cEf1jet today +-9676535 MmMPCF2 tomorrow +-9462165 7WLVW6F4h71Dgk7 tomorrow +-9329892 e7sC5M0H5K6EgSTf41X tomorrow +-9175632 UUBET8444iJDvjUlq3en tomorrow +-9011819 A6CX2HDWN8 tomorrow +-8987676 FhXANp2KDtMmA2gFd778pA tomorrow +-8413710 81Rg5rR0IaInWw tomorrow +-8230445 K6J1LIb5 tomorrow +-7980033 HtI02nss6t8S0fqH4vcLkCD tomorrow +-6882225 r6gCtT4Tgo5rG tomorrow +PREHOOK: query: delete from acid_dwp where a = '-1071363017' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dwp +PREHOOK: Input: default@acid_dwp@ds=today +PREHOOK: Input: default@acid_dwp@ds=tomorrow +PREHOOK: Output: default@acid_dwp@ds=today +PREHOOK: Output: default@acid_dwp@ds=tomorrow +POSTHOOK: query: delete from acid_dwp where a = '-1071363017' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dwp +POSTHOOK: Input: default@acid_dwp@ds=today +POSTHOOK: Input: default@acid_dwp@ds=tomorrow +POSTHOOK: Output: default@acid_dwp@ds=today +POSTHOOK: Output: default@acid_dwp@ds=tomorrow +PREHOOK: query: select * from acid_dwp order by a, ds +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dwp +PREHOOK: Input: default@acid_dwp@ds=today +PREHOOK: Input: default@acid_dwp@ds=tomorrow +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_dwp order by a, ds +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dwp +POSTHOOK: Input: default@acid_dwp@ds=today +POSTHOOK: Input: default@acid_dwp@ds=tomorrow +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa today +-1073051226 A34p7oRr2WvUJNf today +-1072910839 0iqrc5 today +-1072081801 dPkN74F7 today +-1072076362 2uLyD28144vklju213J1mr today +-1071480828 aw724t8c5558x2xneC624 today +-1070883071 0ruyd6Y50JpdGRf6HqD today +-1070551679 iUR3Q today +-1069736047 k17Am8uPHWk02cEf1jet today +-9676535 MmMPCF2 tomorrow +-9462165 7WLVW6F4h71Dgk7 tomorrow +-9329892 e7sC5M0H5K6EgSTf41X tomorrow +-9175632 UUBET8444iJDvjUlq3en tomorrow +-9011819 A6CX2HDWN8 tomorrow +-8987676 FhXANp2KDtMmA2gFd778pA tomorrow +-8413710 81Rg5rR0IaInWw tomorrow +-8230445 K6J1LIb5 tomorrow +-7980033 HtI02nss6t8S0fqH4vcLkCD tomorrow +-6882225 r6gCtT4Tgo5rG tomorrow diff --git ql/src/test/results/clientpositive/tez/delete_whole_partition.q.out ql/src/test/results/clientpositive/tez/delete_whole_partition.q.out new file mode 100644 index 0000000..a2408eb --- /dev/null +++ ql/src/test/results/clientpositive/tez/delete_whole_partition.q.out @@ -0,0 +1,92 @@ +PREHOOK: query: create table acid_dwhp(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_dwhp +POSTHOOK: query: create table acid_dwhp(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_dwhp +PREHOOK: query: insert into table acid_dwhp partition (ds='today') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint < 0 order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_dwhp@ds=today +POSTHOOK: query: insert into table acid_dwhp partition (ds='today') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint < 0 order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_dwhp@ds=today +POSTHOOK: Lineage: acid_dwhp PARTITION(ds=today).a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_dwhp PARTITION(ds=today).b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: insert into table acid_dwhp partition (ds='tomorrow') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint > -10000000 order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_dwhp@ds=tomorrow +POSTHOOK: query: insert into table acid_dwhp partition (ds='tomorrow') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint > -10000000 order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_dwhp@ds=tomorrow +POSTHOOK: Lineage: acid_dwhp PARTITION(ds=tomorrow).a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_dwhp PARTITION(ds=tomorrow).b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: select a,b,ds from acid_dwhp order by a, ds +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dwhp +PREHOOK: Input: default@acid_dwhp@ds=today +PREHOOK: Input: default@acid_dwhp@ds=tomorrow +#### A masked pattern was here #### +POSTHOOK: query: select a,b,ds from acid_dwhp order by a, ds +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dwhp +POSTHOOK: Input: default@acid_dwhp@ds=today +POSTHOOK: Input: default@acid_dwhp@ds=tomorrow +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa today +-1073051226 A34p7oRr2WvUJNf today +-1072910839 0iqrc5 today +-1072081801 dPkN74F7 today +-1072076362 2uLyD28144vklju213J1mr today +-1071480828 aw724t8c5558x2xneC624 today +-1071363017 Anj0oF today +-1070883071 0ruyd6Y50JpdGRf6HqD today +-1070551679 iUR3Q today +-1069736047 k17Am8uPHWk02cEf1jet today +-9676535 MmMPCF2 tomorrow +-9462165 7WLVW6F4h71Dgk7 tomorrow +-9329892 e7sC5M0H5K6EgSTf41X tomorrow +-9175632 UUBET8444iJDvjUlq3en tomorrow +-9011819 A6CX2HDWN8 tomorrow +-8987676 FhXANp2KDtMmA2gFd778pA tomorrow +-8413710 81Rg5rR0IaInWw tomorrow +-8230445 K6J1LIb5 tomorrow +-7980033 HtI02nss6t8S0fqH4vcLkCD tomorrow +-6882225 r6gCtT4Tgo5rG tomorrow +PREHOOK: query: delete from acid_dwhp where ds = 'today' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dwhp +PREHOOK: Input: default@acid_dwhp@ds=today +PREHOOK: Output: default@acid_dwhp@ds=today +POSTHOOK: query: delete from acid_dwhp where ds = 'today' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dwhp +POSTHOOK: Input: default@acid_dwhp@ds=today +POSTHOOK: Output: default@acid_dwhp@ds=today +PREHOOK: query: select * from acid_dwhp order by a, ds +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dwhp +PREHOOK: Input: default@acid_dwhp@ds=today +PREHOOK: Input: default@acid_dwhp@ds=tomorrow +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_dwhp order by a, ds +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dwhp +POSTHOOK: Input: default@acid_dwhp@ds=today +POSTHOOK: Input: default@acid_dwhp@ds=tomorrow +#### A masked pattern was here #### +-9676535 MmMPCF2 tomorrow +-9462165 7WLVW6F4h71Dgk7 tomorrow +-9329892 e7sC5M0H5K6EgSTf41X tomorrow +-9175632 UUBET8444iJDvjUlq3en tomorrow +-9011819 A6CX2HDWN8 tomorrow +-8987676 FhXANp2KDtMmA2gFd778pA tomorrow +-8413710 81Rg5rR0IaInWw tomorrow +-8230445 K6J1LIb5 tomorrow +-7980033 HtI02nss6t8S0fqH4vcLkCD tomorrow +-6882225 r6gCtT4Tgo5rG tomorrow diff --git ql/src/test/results/clientpositive/tez/insert_orig_table.q.out ql/src/test/results/clientpositive/tez/insert_orig_table.q.out new file mode 100644 index 0000000..97a284b --- /dev/null +++ ql/src/test/results/clientpositive/tez/insert_orig_table.q.out @@ -0,0 +1,80 @@ +PREHOOK: query: create table acid_iot( + ctinyint TINYINT, + csmallint SMALLINT, + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE, + cstring1 STRING, + cstring2 STRING, + ctimestamp1 TIMESTAMP, + ctimestamp2 TIMESTAMP, + cboolean1 BOOLEAN, + cboolean2 BOOLEAN) clustered by (cint) into 1 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_iot +POSTHOOK: query: create table acid_iot( + ctinyint TINYINT, + csmallint SMALLINT, + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE, + cstring1 STRING, + cstring2 STRING, + ctimestamp1 TIMESTAMP, + ctimestamp2 TIMESTAMP, + cboolean1 BOOLEAN, + cboolean2 BOOLEAN) clustered by (cint) into 1 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_iot +PREHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/alltypesorc" into table acid_iot +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@acid_iot +POSTHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/alltypesorc" into table acid_iot +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@acid_iot +PREHOOK: query: select count(*) from acid_iot +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_iot +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid_iot +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_iot +#### A masked pattern was here #### +12288 +PREHOOK: query: insert into table acid_iot select ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, + cboolean1, cboolean2 from alltypesorc where cint < 0 order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_iot +POSTHOOK: query: insert into table acid_iot select ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, + cboolean1, cboolean2 from alltypesorc where cint < 0 order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_iot +POSTHOOK: Lineage: acid_iot.cbigint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ] +POSTHOOK: Lineage: acid_iot.cboolean1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean1, type:boolean, comment:null), ] +POSTHOOK: Lineage: acid_iot.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] +POSTHOOK: Lineage: acid_iot.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: acid_iot.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: acid_iot.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_iot.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] +POSTHOOK: Lineage: acid_iot.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: acid_iot.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] +POSTHOOK: Lineage: acid_iot.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] +POSTHOOK: Lineage: acid_iot.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] +POSTHOOK: Lineage: acid_iot.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +PREHOOK: query: select count(*) from acid_iot +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_iot +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid_iot +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_iot +#### A masked pattern was here #### +12298 diff --git ql/src/test/results/clientpositive/tez/insert_update_delete.q.out ql/src/test/results/clientpositive/tez/insert_update_delete.q.out new file mode 100644 index 0000000..e9f9984 --- /dev/null +++ ql/src/test/results/clientpositive/tez/insert_update_delete.q.out @@ -0,0 +1,78 @@ +PREHOOK: query: create table acid_iud(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_iud +POSTHOOK: query: create table acid_iud(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_iud +PREHOOK: query: insert into table acid_iud select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint < 0 order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_iud +POSTHOOK: query: insert into table acid_iud select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint < 0 order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_iud +POSTHOOK: Lineage: acid_iud.a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_iud.b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: select a,b from acid_iud order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_iud +#### A masked pattern was here #### +POSTHOOK: query: select a,b from acid_iud order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_iud +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa +-1073051226 A34p7oRr2WvUJNf +-1072910839 0iqrc5 +-1072081801 dPkN74F7 +-1072076362 2uLyD28144vklju213J1mr +-1071480828 aw724t8c5558x2xneC624 +-1071363017 Anj0oF +-1070883071 0ruyd6Y50JpdGRf6HqD +-1070551679 iUR3Q +-1069736047 k17Am8uPHWk02cEf1jet +PREHOOK: query: update acid_iud set b = 'fred' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_iud +PREHOOK: Output: default@acid_iud +POSTHOOK: query: update acid_iud set b = 'fred' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_iud +POSTHOOK: Output: default@acid_iud +PREHOOK: query: select a,b from acid_iud order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_iud +#### A masked pattern was here #### +POSTHOOK: query: select a,b from acid_iud order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_iud +#### A masked pattern was here #### +-1073279343 fred +-1073051226 fred +-1072910839 fred +-1072081801 fred +-1072076362 fred +-1071480828 fred +-1071363017 fred +-1070883071 fred +-1070551679 fred +-1069736047 fred +PREHOOK: query: delete from acid_iud +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_iud +PREHOOK: Output: default@acid_iud +POSTHOOK: query: delete from acid_iud +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_iud +POSTHOOK: Output: default@acid_iud +PREHOOK: query: select a,b from acid_iud order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_iud +#### A masked pattern was here #### +POSTHOOK: query: select a,b from acid_iud order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_iud +#### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/tez/insert_values_dynamic_partitioned.q.out ql/src/test/results/clientpositive/tez/insert_values_dynamic_partitioned.q.out new file mode 100644 index 0000000..daea059 --- /dev/null +++ ql/src/test/results/clientpositive/tez/insert_values_dynamic_partitioned.q.out @@ -0,0 +1,45 @@ +PREHOOK: query: create table ivdp(i int, + de decimal(5,2), + vc varchar(128)) partitioned by (ds string) clustered by (i) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ivdp +POSTHOOK: query: create table ivdp(i int, + de decimal(5,2), + vc varchar(128)) partitioned by (ds string) clustered by (i) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ivdp +PREHOOK: query: insert into table ivdp partition (ds) values + (1, 109.23, 'and everywhere that mary went', 'today'), + (6553, 923.19, 'the lamb was sure to go', 'tomorrow') +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@ivdp +POSTHOOK: query: insert into table ivdp partition (ds) values + (1, 109.23, 'and everywhere that mary went', 'today'), + (6553, 923.19, 'the lamb was sure to go', 'tomorrow') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@ivdp@ds=today +POSTHOOK: Output: default@ivdp@ds=tomorrow +POSTHOOK: Lineage: ivdp PARTITION(ds=today).de EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: ivdp PARTITION(ds=today).i EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: ivdp PARTITION(ds=today).vc EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: ivdp PARTITION(ds=tomorrow).de EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: ivdp PARTITION(ds=tomorrow).i EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: ivdp PARTITION(ds=tomorrow).vc EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +PREHOOK: query: select * from ivdp order by ds +PREHOOK: type: QUERY +PREHOOK: Input: default@ivdp +PREHOOK: Input: default@ivdp@ds=today +PREHOOK: Input: default@ivdp@ds=tomorrow +#### A masked pattern was here #### +POSTHOOK: query: select * from ivdp order by ds +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ivdp +POSTHOOK: Input: default@ivdp@ds=today +POSTHOOK: Input: default@ivdp@ds=tomorrow +#### A masked pattern was here #### +1 109.23 and everywhere that mary went today +6553 923.19 the lamb was sure to go tomorrow diff --git ql/src/test/results/clientpositive/tez/insert_values_non_partitioned.q.out ql/src/test/results/clientpositive/tez/insert_values_non_partitioned.q.out new file mode 100644 index 0000000..a6f8ac4 --- /dev/null +++ ql/src/test/results/clientpositive/tez/insert_values_non_partitioned.q.out @@ -0,0 +1,64 @@ +PREHOOK: query: create table acid_ivnp(ti tinyint, + si smallint, + i int, + bi bigint, + f float, + d double, + de decimal(5,2), + t timestamp, + dt date, + s string, + vc varchar(128), + ch char(12)) clustered by (i) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_ivnp +POSTHOOK: query: create table acid_ivnp(ti tinyint, + si smallint, + i int, + bi bigint, + f float, + d double, + de decimal(5,2), + t timestamp, + dt date, + s string, + vc varchar(128), + ch char(12)) clustered by (i) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_ivnp +PREHOOK: query: insert into table acid_ivnp values + (1, 257, 65537, 4294967297, 3.14, 3.141592654, 109.23, '2014-08-25 17:21:30.0', '2014-08-25', 'mary had a little lamb', 'ring around the rosie', 'red'), + (3, 25, 6553, 429496729, 0.14, 1923.141592654, 1.2301, '2014-08-24 17:21:30.0', '2014-08-26', 'its fleece was white as snow', 'a pocket full of posies', 'blue' ) +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@acid_ivnp +POSTHOOK: query: insert into table acid_ivnp values + (1, 257, 65537, 4294967297, 3.14, 3.141592654, 109.23, '2014-08-25 17:21:30.0', '2014-08-25', 'mary had a little lamb', 'ring around the rosie', 'red'), + (3, 25, 6553, 429496729, 0.14, 1923.141592654, 1.2301, '2014-08-24 17:21:30.0', '2014-08-26', 'its fleece was white as snow', 'a pocket full of posies', 'blue' ) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@acid_ivnp +POSTHOOK: Lineage: acid_ivnp.bi EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col4, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivnp.ch EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col12, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivnp.d EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col6, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivnp.de EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col7, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivnp.dt EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col9, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivnp.f EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col5, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivnp.i EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivnp.s SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col10, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivnp.si EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivnp.t EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col8, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivnp.ti EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivnp.vc EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col11, type:string, comment:), ] +PREHOOK: query: select ti, si, i, bi, f, d, de, t, dt, s, vc, ch from acid_ivnp order by ti +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_ivnp +#### A masked pattern was here #### +POSTHOOK: query: select ti, si, i, bi, f, d, de, t, dt, s, vc, ch from acid_ivnp order by ti +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_ivnp +#### A masked pattern was here #### +1 257 65537 4294967297 3.14 3.141592654 109.23 2014-08-25 17:21:30 2014-08-25 mary had a little lamb ring around the rosie red +3 25 6553 429496729 0.14 1923.141592654 1.23 2014-08-24 17:21:30 2014-08-26 its fleece was white as snow a pocket full of posies blue diff --git ql/src/test/results/clientpositive/tez/insert_values_orig_table.q.out ql/src/test/results/clientpositive/tez/insert_values_orig_table.q.out new file mode 100644 index 0000000..69220ec --- /dev/null +++ ql/src/test/results/clientpositive/tez/insert_values_orig_table.q.out @@ -0,0 +1,82 @@ +PREHOOK: query: create table acid_ivot( + ctinyint TINYINT, + csmallint SMALLINT, + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE, + cstring1 STRING, + cstring2 STRING, + ctimestamp1 TIMESTAMP, + ctimestamp2 TIMESTAMP, + cboolean1 BOOLEAN, + cboolean2 BOOLEAN) clustered by (cint) into 1 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_ivot +POSTHOOK: query: create table acid_ivot( + ctinyint TINYINT, + csmallint SMALLINT, + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE, + cstring1 STRING, + cstring2 STRING, + ctimestamp1 TIMESTAMP, + ctimestamp2 TIMESTAMP, + cboolean1 BOOLEAN, + cboolean2 BOOLEAN) clustered by (cint) into 1 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_ivot +PREHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/alltypesorc" into table acid_ivot +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@acid_ivot +POSTHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/alltypesorc" into table acid_ivot +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@acid_ivot +PREHOOK: query: select count(*) from acid_ivot +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_ivot +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid_ivot +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_ivot +#### A masked pattern was here #### +12288 +PREHOOK: query: insert into table acid_ivot values + (1, 2, 3, 4, 3.14, 2.34, 'fred', 'bob', '2014-09-01 10:34:23.111', '1944-06-06 06:00:00', true, true), + (111, 222, 3333, 444, 13.14, 10239302.34239320, 'fred', 'bob', '2014-09-01 10:34:23.111', '1944-06-06 06:00:00', true, true) +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@acid_ivot +POSTHOOK: query: insert into table acid_ivot values + (1, 2, 3, 4, 3.14, 2.34, 'fred', 'bob', '2014-09-01 10:34:23.111', '1944-06-06 06:00:00', true, true), + (111, 222, 3333, 444, 13.14, 10239302.34239320, 'fred', 'bob', '2014-09-01 10:34:23.111', '1944-06-06 06:00:00', true, true) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@acid_ivot +POSTHOOK: Lineage: acid_ivot.cbigint EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col4, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.cboolean1 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col11, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.cboolean2 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col12, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.cdouble EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col6, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.cfloat EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col5, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.cint EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.csmallint EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.cstring1 SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col7, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.cstring2 SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col8, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.ctimestamp1 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col9, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.ctimestamp2 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col10, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.ctinyint EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: select count(*) from acid_ivot +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_ivot +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid_ivot +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_ivot +#### A masked pattern was here #### +12290 diff --git ql/src/test/results/clientpositive/tez/insert_values_partitioned.q.out ql/src/test/results/clientpositive/tez/insert_values_partitioned.q.out new file mode 100644 index 0000000..9fb89ff --- /dev/null +++ ql/src/test/results/clientpositive/tez/insert_values_partitioned.q.out @@ -0,0 +1,66 @@ +PREHOOK: query: create table acid_ivp(ti tinyint, + si smallint, + i int, + bi bigint, + f float, + d double, + de decimal(5,2), + t timestamp, + dt date, + s string, + vc varchar(128), + ch char(12)) partitioned by (ds string) clustered by (i) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_ivp +POSTHOOK: query: create table acid_ivp(ti tinyint, + si smallint, + i int, + bi bigint, + f float, + d double, + de decimal(5,2), + t timestamp, + dt date, + s string, + vc varchar(128), + ch char(12)) partitioned by (ds string) clustered by (i) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_ivp +PREHOOK: query: insert into table acid_ivp partition (ds='today') values + (1, 257, 65537, 4294967297, 3.14, 3.141592654, 109.23, '2014-08-25 17:21:30.0', '2014-08-25', 'mary had a little lamb', 'ring around the rosie', 'red'), + (3, 25, 6553, 429496729, 0.14, 1923.141592654, 1.2301, '2014-08-24 17:21:30.0', '2014-08-26', 'its fleece was white as snow', 'a pocket full of posies', 'blue') +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@acid_ivp@ds=today +POSTHOOK: query: insert into table acid_ivp partition (ds='today') values + (1, 257, 65537, 4294967297, 3.14, 3.141592654, 109.23, '2014-08-25 17:21:30.0', '2014-08-25', 'mary had a little lamb', 'ring around the rosie', 'red'), + (3, 25, 6553, 429496729, 0.14, 1923.141592654, 1.2301, '2014-08-24 17:21:30.0', '2014-08-26', 'its fleece was white as snow', 'a pocket full of posies', 'blue') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@acid_ivp@ds=today +POSTHOOK: Lineage: acid_ivp PARTITION(ds=today).bi EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col4, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivp PARTITION(ds=today).ch EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col12, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivp PARTITION(ds=today).d EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col6, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivp PARTITION(ds=today).de EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col7, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivp PARTITION(ds=today).dt EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col9, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivp PARTITION(ds=today).f EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col5, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivp PARTITION(ds=today).i EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivp PARTITION(ds=today).s SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col10, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivp PARTITION(ds=today).si EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivp PARTITION(ds=today).t EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col8, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivp PARTITION(ds=today).ti EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivp PARTITION(ds=today).vc EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col11, type:string, comment:), ] +PREHOOK: query: select * from acid_ivp order by i +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_ivp +PREHOOK: Input: default@acid_ivp@ds=today +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_ivp order by i +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_ivp +POSTHOOK: Input: default@acid_ivp@ds=today +#### A masked pattern was here #### +3 25 6553 429496729 0.14 1923.141592654 1.23 2014-08-24 17:21:30 2014-08-26 its fleece was white as snow a pocket full of posies blue today +1 257 65537 4294967297 3.14 3.141592654 109.23 2014-08-25 17:21:30 2014-08-25 mary had a little lamb ring around the rosie red today diff --git ql/src/test/results/clientpositive/tez/insert_values_tmp_table.q.out ql/src/test/results/clientpositive/tez/insert_values_tmp_table.q.out new file mode 100644 index 0000000..95d6372 --- /dev/null +++ ql/src/test/results/clientpositive/tez/insert_values_tmp_table.q.out @@ -0,0 +1,33 @@ +PREHOOK: query: create temporary table acid_ivtt(i int, de decimal(5,2), vc varchar(128)) clustered by (vc) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_ivtt +POSTHOOK: query: create temporary table acid_ivtt(i int, de decimal(5,2), vc varchar(128)) clustered by (vc) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_ivtt +PREHOOK: query: insert into table acid_ivtt values + (1, 109.23, 'mary had a little lamb'), + (429496729, 0.14, 'its fleece was white as snow') +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@acid_ivtt +POSTHOOK: query: insert into table acid_ivtt values + (1, 109.23, 'mary had a little lamb'), + (429496729, 0.14, 'its fleece was white as snow') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@acid_ivtt +POSTHOOK: Lineage: acid_ivtt.de EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivtt.i EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivtt.vc EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +PREHOOK: query: select i, de, vc from acid_ivtt order by i +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_ivtt +#### A masked pattern was here #### +POSTHOOK: query: select i, de, vc from acid_ivtt order by i +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_ivtt +#### A masked pattern was here #### +1 109.23 mary had a little lamb +429496729 0.14 its fleece was white as snow diff --git ql/src/test/results/clientpositive/tez/update_after_multiple_inserts.q.out ql/src/test/results/clientpositive/tez/update_after_multiple_inserts.q.out new file mode 100644 index 0000000..cb8e319 --- /dev/null +++ ql/src/test/results/clientpositive/tez/update_after_multiple_inserts.q.out @@ -0,0 +1,74 @@ +PREHOOK: query: create table acid_uami(i int, + de decimal(5,2), + vc varchar(128)) clustered by (i) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_uami +POSTHOOK: query: create table acid_uami(i int, + de decimal(5,2), + vc varchar(128)) clustered by (i) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_uami +PREHOOK: query: insert into table acid_uami values + (1, 109.23, 'mary had a little lamb'), + (6553, 923.19, 'its fleece was white as snow') +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@acid_uami +POSTHOOK: query: insert into table acid_uami values + (1, 109.23, 'mary had a little lamb'), + (6553, 923.19, 'its fleece was white as snow') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@acid_uami +POSTHOOK: Lineage: acid_uami.de EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: acid_uami.i EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: acid_uami.vc EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +PREHOOK: query: insert into table acid_uami values + (10, 119.23, 'and everywhere that mary went'), + (65530, 823.19, 'the lamb was sure to go') +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__2 +PREHOOK: Output: default@acid_uami +POSTHOOK: query: insert into table acid_uami values + (10, 119.23, 'and everywhere that mary went'), + (65530, 823.19, 'the lamb was sure to go') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__2 +POSTHOOK: Output: default@acid_uami +POSTHOOK: Lineage: acid_uami.de EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: acid_uami.i EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: acid_uami.vc EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +PREHOOK: query: select * from acid_uami order by de +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uami +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_uami order by de +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uami +#### A masked pattern was here #### +1 109.23 mary had a little lamb +10 119.23 and everywhere that mary went +65530 823.19 the lamb was sure to go +6553 923.19 its fleece was white as snow +PREHOOK: query: update acid_uami set de = 3.14 where de = 109.23 or de = 119.23 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uami +PREHOOK: Output: default@acid_uami +POSTHOOK: query: update acid_uami set de = 3.14 where de = 109.23 or de = 119.23 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uami +POSTHOOK: Output: default@acid_uami +PREHOOK: query: select * from acid_uami order by de +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uami +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_uami order by de +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uami +#### A masked pattern was here #### +1 3.14 mary had a little lamb +10 3.14 and everywhere that mary went +65530 823.19 the lamb was sure to go +6553 923.19 its fleece was white as snow diff --git ql/src/test/results/clientpositive/tez/update_all_non_partitioned.q.out ql/src/test/results/clientpositive/tez/update_all_non_partitioned.q.out new file mode 100644 index 0000000..fde6d8d --- /dev/null +++ ql/src/test/results/clientpositive/tez/update_all_non_partitioned.q.out @@ -0,0 +1,62 @@ +PREHOOK: query: create table acid_uanp(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_uanp +POSTHOOK: query: create table acid_uanp(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_uanp +PREHOOK: query: insert into table acid_uanp select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint < 0 order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_uanp +POSTHOOK: query: insert into table acid_uanp select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint < 0 order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_uanp +POSTHOOK: Lineage: acid_uanp.a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_uanp.b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: select a,b from acid_uanp order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uanp +#### A masked pattern was here #### +POSTHOOK: query: select a,b from acid_uanp order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uanp +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa +-1073051226 A34p7oRr2WvUJNf +-1072910839 0iqrc5 +-1072081801 dPkN74F7 +-1072076362 2uLyD28144vklju213J1mr +-1071480828 aw724t8c5558x2xneC624 +-1071363017 Anj0oF +-1070883071 0ruyd6Y50JpdGRf6HqD +-1070551679 iUR3Q +-1069736047 k17Am8uPHWk02cEf1jet +PREHOOK: query: update acid_uanp set b = 'fred' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uanp +PREHOOK: Output: default@acid_uanp +POSTHOOK: query: update acid_uanp set b = 'fred' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uanp +POSTHOOK: Output: default@acid_uanp +PREHOOK: query: select a,b from acid_uanp order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uanp +#### A masked pattern was here #### +POSTHOOK: query: select a,b from acid_uanp order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uanp +#### A masked pattern was here #### +-1073279343 fred +-1073051226 fred +-1072910839 fred +-1072081801 fred +-1072076362 fred +-1071480828 fred +-1071363017 fred +-1070883071 fred +-1070551679 fred +-1069736047 fred diff --git ql/src/test/results/clientpositive/tez/update_all_partitioned.q.out ql/src/test/results/clientpositive/tez/update_all_partitioned.q.out new file mode 100644 index 0000000..3fae6a9 --- /dev/null +++ ql/src/test/results/clientpositive/tez/update_all_partitioned.q.out @@ -0,0 +1,106 @@ +PREHOOK: query: create table acid_uap(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_uap +POSTHOOK: query: create table acid_uap(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_uap +PREHOOK: query: insert into table acid_uap partition (ds='today') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint < 0 order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_uap@ds=today +POSTHOOK: query: insert into table acid_uap partition (ds='today') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint < 0 order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_uap@ds=today +POSTHOOK: Lineage: acid_uap PARTITION(ds=today).a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_uap PARTITION(ds=today).b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: insert into table acid_uap partition (ds='tomorrow') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint > 10 order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_uap@ds=tomorrow +POSTHOOK: query: insert into table acid_uap partition (ds='tomorrow') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint > 10 order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_uap@ds=tomorrow +POSTHOOK: Lineage: acid_uap PARTITION(ds=tomorrow).a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_uap PARTITION(ds=tomorrow).b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: select a,b,ds from acid_uap order by a,b +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uap +PREHOOK: Input: default@acid_uap@ds=today +PREHOOK: Input: default@acid_uap@ds=tomorrow +#### A masked pattern was here #### +POSTHOOK: query: select a,b,ds from acid_uap order by a,b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uap +POSTHOOK: Input: default@acid_uap@ds=today +POSTHOOK: Input: default@acid_uap@ds=tomorrow +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa today +-1073051226 A34p7oRr2WvUJNf today +-1072910839 0iqrc5 today +-1072081801 dPkN74F7 today +-1072076362 2uLyD28144vklju213J1mr today +-1071480828 aw724t8c5558x2xneC624 today +-1071363017 Anj0oF today +-1070883071 0ruyd6Y50JpdGRf6HqD today +-1070551679 iUR3Q today +-1069736047 k17Am8uPHWk02cEf1jet today +762 3WsVeqb28VWEEOLI8ail tomorrow +762 40ks5556SV tomorrow +762 BLoMwUJ51ns6pd tomorrow +762 a10E76jX35YwquKCTA tomorrow +762 q5y2Vy1 tomorrow +6981 1FNNhmiFLGw425NA13g tomorrow +6981 K630vaVf tomorrow +6981 YdG61y00526u5 tomorrow +6981 a3EhVU6Wuy7ycJ7wY7h2gv tomorrow +6981 o4lvY20511w0EOX3P3I82p63 tomorrow +PREHOOK: query: update acid_uap set b = 'fred' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uap +PREHOOK: Input: default@acid_uap@ds=today +PREHOOK: Input: default@acid_uap@ds=tomorrow +PREHOOK: Output: default@acid_uap@ds=today +PREHOOK: Output: default@acid_uap@ds=tomorrow +POSTHOOK: query: update acid_uap set b = 'fred' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uap +POSTHOOK: Input: default@acid_uap@ds=today +POSTHOOK: Input: default@acid_uap@ds=tomorrow +POSTHOOK: Output: default@acid_uap@ds=today +POSTHOOK: Output: default@acid_uap@ds=tomorrow +PREHOOK: query: select a,b,ds from acid_uap order by a,b +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uap +PREHOOK: Input: default@acid_uap@ds=today +PREHOOK: Input: default@acid_uap@ds=tomorrow +#### A masked pattern was here #### +POSTHOOK: query: select a,b,ds from acid_uap order by a,b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uap +POSTHOOK: Input: default@acid_uap@ds=today +POSTHOOK: Input: default@acid_uap@ds=tomorrow +#### A masked pattern was here #### +-1073279343 fred today +-1073051226 fred today +-1072910839 fred today +-1072081801 fred today +-1072076362 fred today +-1071480828 fred today +-1071363017 fred today +-1070883071 fred today +-1070551679 fred today +-1069736047 fred today +762 fred tomorrow +762 fred tomorrow +762 fred tomorrow +762 fred tomorrow +762 fred tomorrow +6981 fred tomorrow +6981 fred tomorrow +6981 fred tomorrow +6981 fred tomorrow +6981 fred tomorrow diff --git ql/src/test/results/clientpositive/tez/update_all_types.q.out ql/src/test/results/clientpositive/tez/update_all_types.q.out new file mode 100644 index 0000000..36b4684 --- /dev/null +++ ql/src/test/results/clientpositive/tez/update_all_types.q.out @@ -0,0 +1,153 @@ +PREHOOK: query: create table acid_uat(ti tinyint, + si smallint, + i int, + bi bigint, + f float, + d double, + de decimal(5,2), + t timestamp, + dt date, + s string, + vc varchar(128), + ch char(36), + b boolean) clustered by (i) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_uat +POSTHOOK: query: create table acid_uat(ti tinyint, + si smallint, + i int, + bi bigint, + f float, + d double, + de decimal(5,2), + t timestamp, + dt date, + s string, + vc varchar(128), + ch char(36), + b boolean) clustered by (i) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_uat +PREHOOK: query: insert into table acid_uat + select ctinyint, + csmallint, + cint, + cbigint, + cfloat, + cdouble, + cast(cfloat as decimal(5,2)), + ctimestamp1, + cast(ctimestamp2 as date), + cstring1, + cast(cstring1 as varchar(128)), + cast(cstring2 as char(36)), + cboolean1 + from alltypesorc where cint < 0 order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_uat +POSTHOOK: query: insert into table acid_uat + select ctinyint, + csmallint, + cint, + cbigint, + cfloat, + cdouble, + cast(cfloat as decimal(5,2)), + ctimestamp1, + cast(ctimestamp2 as date), + cstring1, + cast(cstring1 as varchar(128)), + cast(cstring2 as char(36)), + cboolean1 + from alltypesorc where cint < 0 order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_uat +POSTHOOK: Lineage: acid_uat.b SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean1, type:boolean, comment:null), ] +POSTHOOK: Lineage: acid_uat.bi SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ] +POSTHOOK: Lineage: acid_uat.ch EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] +POSTHOOK: Lineage: acid_uat.d SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: acid_uat.de EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: acid_uat.dt EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] +POSTHOOK: Lineage: acid_uat.f SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: acid_uat.i SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_uat.s SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: acid_uat.si SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] +POSTHOOK: Lineage: acid_uat.t SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] +POSTHOOK: Lineage: acid_uat.ti SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +POSTHOOK: Lineage: acid_uat.vc EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: select * from acid_uat order by i +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uat +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_uat order by i +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uat +#### A masked pattern was here #### +11 NULL -1073279343 -1595604468 11.0 NULL 11.0 1969-12-31 16:00:02.351 NULL oj1YrV5Wa oj1YrV5Wa P76636jJ6qM17d7DIy true +NULL -7382 -1073051226 -1887561756 NULL -7382.0 NULL NULL 1970-01-01 A34p7oRr2WvUJNf A34p7oRr2WvUJNf 4hA4KQj2vD3fI6gX82220d false +11 NULL -1072910839 2048385991 11.0 NULL 11.0 1969-12-31 16:00:02.351 NULL 0iqrc5 0iqrc5 KbaDXiN85adbHRx58v false +NULL 8373 -1072081801 1864027286 NULL 8373.0 NULL NULL 1970-01-01 dPkN74F7 dPkN74F7 4KWs6gw7lv2WYd66P true +NULL -5470 -1072076362 1864027286 NULL -5470.0 NULL NULL 1970-01-01 2uLyD28144vklju213J1mr 2uLyD28144vklju213J1mr 4KWs6gw7lv2WYd66P true +-51 NULL -1071480828 -1401575336 -51.0 NULL -51.0 1969-12-31 16:00:08.451 NULL aw724t8c5558x2xneC624 aw724t8c5558x2xneC624 4uE7l74tESBiKfu7c8wM7GA true +8 NULL -1071363017 1349676361 8.0 NULL 8.0 1969-12-31 16:00:15.892 NULL Anj0oF Anj0oF IwE1G7Qb0B1NEfV030g true +NULL -741 -1070883071 -1645852809 NULL -741.0 NULL NULL 1970-01-01 0ruyd6Y50JpdGRf6HqD 0ruyd6Y50JpdGRf6HqD xH7445Rals48VOulSyR5F false +NULL -947 -1070551679 1864027286 NULL -947.0 NULL NULL 1970-01-01 iUR3Q iUR3Q 4KWs6gw7lv2WYd66P false +11 NULL -1069736047 -453772520 11.0 NULL 11.0 1969-12-31 16:00:02.351 NULL k17Am8uPHWk02cEf1jet k17Am8uPHWk02cEf1jet qrXLLNX1 true +PREHOOK: query: update acid_uat set + ti = 1, + si = 2, + i = 3, + bi = 4, + f = 3.14, + d = 6.28, + de = 5.99, + t = '2014-09-01 09:44.23.23', + dt = '2014-09-01', + s = 'its a beautiful day in the neighbhorhood', + vc = 'a beautiful day for a neighbor', + ch = 'wont you be mine', + b = true + where s = '0ruyd6Y50JpdGRf6HqD' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uat +PREHOOK: Output: default@acid_uat +POSTHOOK: query: update acid_uat set + ti = 1, + si = 2, + i = 3, + bi = 4, + f = 3.14, + d = 6.28, + de = 5.99, + t = '2014-09-01 09:44.23.23', + dt = '2014-09-01', + s = 'its a beautiful day in the neighbhorhood', + vc = 'a beautiful day for a neighbor', + ch = 'wont you be mine', + b = true + where s = '0ruyd6Y50JpdGRf6HqD' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uat +POSTHOOK: Output: default@acid_uat +PREHOOK: query: select * from acid_uat order by i +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uat +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_uat order by i +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uat +#### A masked pattern was here #### +11 NULL -1073279343 -1595604468 11.0 NULL 11.0 1969-12-31 16:00:02.351 NULL oj1YrV5Wa oj1YrV5Wa P76636jJ6qM17d7DIy true +NULL -7382 -1073051226 -1887561756 NULL -7382.0 NULL NULL 1970-01-01 A34p7oRr2WvUJNf A34p7oRr2WvUJNf 4hA4KQj2vD3fI6gX82220d false +11 NULL -1072910839 2048385991 11.0 NULL 11.0 1969-12-31 16:00:02.351 NULL 0iqrc5 0iqrc5 KbaDXiN85adbHRx58v false +NULL 8373 -1072081801 1864027286 NULL 8373.0 NULL NULL 1970-01-01 dPkN74F7 dPkN74F7 4KWs6gw7lv2WYd66P true +NULL -5470 -1072076362 1864027286 NULL -5470.0 NULL NULL 1970-01-01 2uLyD28144vklju213J1mr 2uLyD28144vklju213J1mr 4KWs6gw7lv2WYd66P true +-51 NULL -1071480828 -1401575336 -51.0 NULL -51.0 1969-12-31 16:00:08.451 NULL aw724t8c5558x2xneC624 aw724t8c5558x2xneC624 4uE7l74tESBiKfu7c8wM7GA true +8 NULL -1071363017 1349676361 8.0 NULL 8.0 1969-12-31 16:00:15.892 NULL Anj0oF Anj0oF IwE1G7Qb0B1NEfV030g true +NULL -947 -1070551679 1864027286 NULL -947.0 NULL NULL 1970-01-01 iUR3Q iUR3Q 4KWs6gw7lv2WYd66P false +11 NULL -1069736047 -453772520 11.0 NULL 11.0 1969-12-31 16:00:02.351 NULL k17Am8uPHWk02cEf1jet k17Am8uPHWk02cEf1jet qrXLLNX1 true +1 2 3 4 3.14 6.28 5.99 NULL 2014-09-01 its a beautiful day in the neighbhorhood a beautiful day for a neighbor wont you be mine true diff --git ql/src/test/results/clientpositive/tez/update_orig_table.q.out ql/src/test/results/clientpositive/tez/update_orig_table.q.out new file mode 100644 index 0000000..de7dc86 --- /dev/null +++ ql/src/test/results/clientpositive/tez/update_orig_table.q.out @@ -0,0 +1,59 @@ +PREHOOK: query: create table acid_uot( + ctinyint TINYINT, + csmallint SMALLINT, + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE, + cstring1 STRING, + cstring2 STRING, + ctimestamp1 TIMESTAMP, + ctimestamp2 TIMESTAMP, + cboolean1 BOOLEAN, +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_uot +POSTHOOK: query: create table acid_uot( + ctinyint TINYINT, + csmallint SMALLINT, + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE, + cstring1 STRING, + cstring2 STRING, + ctimestamp1 TIMESTAMP, + ctimestamp2 TIMESTAMP, + cboolean1 BOOLEAN, +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_uot +PREHOOK: query: update acid_uot set cstring1 = 'fred' where cint < -1070551679 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uot +PREHOOK: Output: default@acid_uot +POSTHOOK: query: update acid_uot set cstring1 = 'fred' where cint < -1070551679 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uot +POSTHOOK: Output: default@acid_uot +PREHOOK: query: select * from acid_uot where cstring1 = 'fred' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uot +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_uot where cstring1 = 'fred' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uot +#### A masked pattern was here #### +-51 NULL -1071480828 -1401575336 -51.0 NULL fred 4uE7l74tESBiKfu7c8wM7GA 1969-12-31 16:00:08.451 NULL true true +NULL -5470 -1072076362 1864027286 NULL -5470.0 fred 4KWs6gw7lv2WYd66P NULL 1969-12-31 16:00:01.836 true true +NULL 8373 -1072081801 1864027286 NULL 8373.0 fred 4KWs6gw7lv2WYd66P NULL 1969-12-31 15:59:56.465 true true +8 NULL -1071363017 1349676361 8.0 NULL fred IwE1G7Qb0B1NEfV030g 1969-12-31 16:00:15.892 NULL true true +NULL -7382 -1073051226 -1887561756 NULL -7382.0 fred 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:10.331 false false +11 NULL -1073279343 -1595604468 11.0 NULL fred P76636jJ6qM17d7DIy 1969-12-31 16:00:02.351 NULL true true +11 NULL -1072910839 2048385991 11.0 NULL fred KbaDXiN85adbHRx58v 1969-12-31 16:00:02.351 NULL false false +NULL -741 -1070883071 -1645852809 NULL -741.0 fred xH7445Rals48VOulSyR5F NULL 1969-12-31 15:59:51.293 false false +#### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/tez/update_tmp_table.q.out ql/src/test/results/clientpositive/tez/update_tmp_table.q.out new file mode 100644 index 0000000..8180f06 --- /dev/null +++ ql/src/test/results/clientpositive/tez/update_tmp_table.q.out @@ -0,0 +1,62 @@ +PREHOOK: query: create table acid_utt(a int, b varchar(128)) clustered by (b) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_utt +POSTHOOK: query: create table acid_utt(a int, b varchar(128)) clustered by (b) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_utt +PREHOOK: query: insert into table acid_utt select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_utt +POSTHOOK: query: insert into table acid_utt select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_utt +POSTHOOK: Lineage: acid_utt.a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_utt.b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: select a,b from acid_utt order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_utt +#### A masked pattern was here #### +POSTHOOK: query: select a,b from acid_utt order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_utt +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa +-1073051226 A34p7oRr2WvUJNf +-1072910839 0iqrc5 +-1072081801 dPkN74F7 +-1072076362 2uLyD28144vklju213J1mr +-1071480828 aw724t8c5558x2xneC624 +-1071363017 Anj0oF +-1070883071 0ruyd6Y50JpdGRf6HqD +-1070551679 iUR3Q +-1069736047 k17Am8uPHWk02cEf1jet +PREHOOK: query: update acid_utt set b = 'fred' where b = '0ruyd6Y50JpdGRf6HqD' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_utt +PREHOOK: Output: default@acid_utt +POSTHOOK: query: update acid_utt set b = 'fred' where b = '0ruyd6Y50JpdGRf6HqD' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_utt +POSTHOOK: Output: default@acid_utt +PREHOOK: query: select * from acid_utt order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_utt +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_utt order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_utt +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa +-1073051226 A34p7oRr2WvUJNf +-1072910839 0iqrc5 +-1072081801 dPkN74F7 +-1072076362 2uLyD28144vklju213J1mr +-1071480828 aw724t8c5558x2xneC624 +-1071363017 Anj0oF +-1070883071 fred +-1070551679 iUR3Q +-1069736047 k17Am8uPHWk02cEf1jet diff --git ql/src/test/results/clientpositive/tez/update_two_cols.q.out ql/src/test/results/clientpositive/tez/update_two_cols.q.out new file mode 100644 index 0000000..553608f --- /dev/null +++ ql/src/test/results/clientpositive/tez/update_two_cols.q.out @@ -0,0 +1,63 @@ +PREHOOK: query: create table acid_utc(a int, b varchar(128), c float) clustered by (a) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_utc +POSTHOOK: query: create table acid_utc(a int, b varchar(128), c float) clustered by (a) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_utc +PREHOOK: query: insert into table acid_utc select cint, cast(cstring1 as varchar(128)), cfloat from alltypesorc where cint < 0 order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_utc +POSTHOOK: query: insert into table acid_utc select cint, cast(cstring1 as varchar(128)), cfloat from alltypesorc where cint < 0 order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_utc +POSTHOOK: Lineage: acid_utc.a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_utc.b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: acid_utc.c SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] +PREHOOK: query: select * from acid_utc order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_utc +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_utc order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_utc +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa 11.0 +-1073051226 A34p7oRr2WvUJNf NULL +-1072910839 0iqrc5 11.0 +-1072081801 dPkN74F7 NULL +-1072076362 2uLyD28144vklju213J1mr NULL +-1071480828 aw724t8c5558x2xneC624 -51.0 +-1071363017 Anj0oF 8.0 +-1070883071 0ruyd6Y50JpdGRf6HqD NULL +-1070551679 iUR3Q NULL +-1069736047 k17Am8uPHWk02cEf1jet 11.0 +PREHOOK: query: update acid_utc set b = 'fred',c = 3.14 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_utc +PREHOOK: Output: default@acid_utc +POSTHOOK: query: update acid_utc set b = 'fred',c = 3.14 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_utc +POSTHOOK: Output: default@acid_utc +PREHOOK: query: select * from acid_utc order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_utc +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_utc order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_utc +#### A masked pattern was here #### +-1073279343 fred 3.14 +-1073051226 fred 3.14 +-1072910839 fred 3.14 +-1072081801 fred 3.14 +-1072076362 fred 3.14 +-1071480828 fred 3.14 +-1071363017 fred 3.14 +-1070883071 fred 3.14 +-1070551679 fred 3.14 +-1069736047 fred 3.14 diff --git ql/src/test/results/clientpositive/tez/update_where_no_match.q.out ql/src/test/results/clientpositive/tez/update_where_no_match.q.out new file mode 100644 index 0000000..afef267 --- /dev/null +++ ql/src/test/results/clientpositive/tez/update_where_no_match.q.out @@ -0,0 +1,62 @@ +PREHOOK: query: create table acid_wnm(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_wnm +POSTHOOK: query: create table acid_wnm(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_wnm +PREHOOK: query: insert into table acid_wnm select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_wnm +POSTHOOK: query: insert into table acid_wnm select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_wnm +POSTHOOK: Lineage: acid_wnm.a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_wnm.b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: select a,b from acid_wnm order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_wnm +#### A masked pattern was here #### +POSTHOOK: query: select a,b from acid_wnm order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_wnm +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa +-1073051226 A34p7oRr2WvUJNf +-1072910839 0iqrc5 +-1072081801 dPkN74F7 +-1072076362 2uLyD28144vklju213J1mr +-1071480828 aw724t8c5558x2xneC624 +-1071363017 Anj0oF +-1070883071 0ruyd6Y50JpdGRf6HqD +-1070551679 iUR3Q +-1069736047 k17Am8uPHWk02cEf1jet +PREHOOK: query: update acid_wnm set b = 'fred' where b = 'nosuchvalue' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_wnm +PREHOOK: Output: default@acid_wnm +POSTHOOK: query: update acid_wnm set b = 'fred' where b = 'nosuchvalue' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_wnm +POSTHOOK: Output: default@acid_wnm +PREHOOK: query: select * from acid_wnm order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_wnm +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_wnm order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_wnm +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa +-1073051226 A34p7oRr2WvUJNf +-1072910839 0iqrc5 +-1072081801 dPkN74F7 +-1072076362 2uLyD28144vklju213J1mr +-1071480828 aw724t8c5558x2xneC624 +-1071363017 Anj0oF +-1070883071 0ruyd6Y50JpdGRf6HqD +-1070551679 iUR3Q +-1069736047 k17Am8uPHWk02cEf1jet diff --git ql/src/test/results/clientpositive/tez/update_where_non_partitioned.q.out ql/src/test/results/clientpositive/tez/update_where_non_partitioned.q.out new file mode 100644 index 0000000..5c79379 --- /dev/null +++ ql/src/test/results/clientpositive/tez/update_where_non_partitioned.q.out @@ -0,0 +1,62 @@ +PREHOOK: query: create table acid_uwnp(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_uwnp +POSTHOOK: query: create table acid_uwnp(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_uwnp +PREHOOK: query: insert into table acid_uwnp select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_uwnp +POSTHOOK: query: insert into table acid_uwnp select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_uwnp +POSTHOOK: Lineage: acid_uwnp.a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_uwnp.b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: select a,b from acid_uwnp order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uwnp +#### A masked pattern was here #### +POSTHOOK: query: select a,b from acid_uwnp order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uwnp +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa +-1073051226 A34p7oRr2WvUJNf +-1072910839 0iqrc5 +-1072081801 dPkN74F7 +-1072076362 2uLyD28144vklju213J1mr +-1071480828 aw724t8c5558x2xneC624 +-1071363017 Anj0oF +-1070883071 0ruyd6Y50JpdGRf6HqD +-1070551679 iUR3Q +-1069736047 k17Am8uPHWk02cEf1jet +PREHOOK: query: update acid_uwnp set b = 'fred' where b = '0ruyd6Y50JpdGRf6HqD' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uwnp +PREHOOK: Output: default@acid_uwnp +POSTHOOK: query: update acid_uwnp set b = 'fred' where b = '0ruyd6Y50JpdGRf6HqD' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uwnp +POSTHOOK: Output: default@acid_uwnp +PREHOOK: query: select * from acid_uwnp order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uwnp +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_uwnp order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uwnp +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa +-1073051226 A34p7oRr2WvUJNf +-1072910839 0iqrc5 +-1072081801 dPkN74F7 +-1072076362 2uLyD28144vklju213J1mr +-1071480828 aw724t8c5558x2xneC624 +-1071363017 Anj0oF +-1070883071 fred +-1070551679 iUR3Q +-1069736047 k17Am8uPHWk02cEf1jet diff --git ql/src/test/results/clientpositive/tez/update_where_partitioned.q.out ql/src/test/results/clientpositive/tez/update_where_partitioned.q.out new file mode 100644 index 0000000..b83c52a --- /dev/null +++ ql/src/test/results/clientpositive/tez/update_where_partitioned.q.out @@ -0,0 +1,106 @@ +PREHOOK: query: create table acid_uwp(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_uwp +POSTHOOK: query: create table acid_uwp(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_uwp +PREHOOK: query: insert into table acid_uwp partition (ds='today') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint < 0 order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_uwp@ds=today +POSTHOOK: query: insert into table acid_uwp partition (ds='today') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint < 0 order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_uwp@ds=today +POSTHOOK: Lineage: acid_uwp PARTITION(ds=today).a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_uwp PARTITION(ds=today).b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: insert into table acid_uwp partition (ds='tomorrow') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint > 100 order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_uwp@ds=tomorrow +POSTHOOK: query: insert into table acid_uwp partition (ds='tomorrow') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint > 100 order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_uwp@ds=tomorrow +POSTHOOK: Lineage: acid_uwp PARTITION(ds=tomorrow).a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_uwp PARTITION(ds=tomorrow).b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: select a,b,ds from acid_uwp order by a, ds, b +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uwp +PREHOOK: Input: default@acid_uwp@ds=today +PREHOOK: Input: default@acid_uwp@ds=tomorrow +#### A masked pattern was here #### +POSTHOOK: query: select a,b,ds from acid_uwp order by a, ds, b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uwp +POSTHOOK: Input: default@acid_uwp@ds=today +POSTHOOK: Input: default@acid_uwp@ds=tomorrow +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa today +-1073051226 A34p7oRr2WvUJNf today +-1072910839 0iqrc5 today +-1072081801 dPkN74F7 today +-1072076362 2uLyD28144vklju213J1mr today +-1071480828 aw724t8c5558x2xneC624 today +-1071363017 Anj0oF today +-1070883071 0ruyd6Y50JpdGRf6HqD today +-1070551679 iUR3Q today +-1069736047 k17Am8uPHWk02cEf1jet today +762 3WsVeqb28VWEEOLI8ail tomorrow +762 40ks5556SV tomorrow +762 BLoMwUJ51ns6pd tomorrow +762 a10E76jX35YwquKCTA tomorrow +762 q5y2Vy1 tomorrow +6981 1FNNhmiFLGw425NA13g tomorrow +6981 K630vaVf tomorrow +6981 YdG61y00526u5 tomorrow +6981 a3EhVU6Wuy7ycJ7wY7h2gv tomorrow +6981 o4lvY20511w0EOX3P3I82p63 tomorrow +PREHOOK: query: update acid_uwp set b = 'fred' where b = 'k17Am8uPHWk02cEf1jet' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uwp +PREHOOK: Input: default@acid_uwp@ds=today +PREHOOK: Input: default@acid_uwp@ds=tomorrow +PREHOOK: Output: default@acid_uwp@ds=today +PREHOOK: Output: default@acid_uwp@ds=tomorrow +POSTHOOK: query: update acid_uwp set b = 'fred' where b = 'k17Am8uPHWk02cEf1jet' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uwp +POSTHOOK: Input: default@acid_uwp@ds=today +POSTHOOK: Input: default@acid_uwp@ds=tomorrow +POSTHOOK: Output: default@acid_uwp@ds=today +POSTHOOK: Output: default@acid_uwp@ds=tomorrow +PREHOOK: query: select * from acid_uwp order by a, ds, b +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uwp +PREHOOK: Input: default@acid_uwp@ds=today +PREHOOK: Input: default@acid_uwp@ds=tomorrow +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_uwp order by a, ds, b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uwp +POSTHOOK: Input: default@acid_uwp@ds=today +POSTHOOK: Input: default@acid_uwp@ds=tomorrow +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa today +-1073051226 A34p7oRr2WvUJNf today +-1072910839 0iqrc5 today +-1072081801 dPkN74F7 today +-1072076362 2uLyD28144vklju213J1mr today +-1071480828 aw724t8c5558x2xneC624 today +-1071363017 Anj0oF today +-1070883071 0ruyd6Y50JpdGRf6HqD today +-1070551679 iUR3Q today +-1069736047 fred today +762 3WsVeqb28VWEEOLI8ail tomorrow +762 40ks5556SV tomorrow +762 BLoMwUJ51ns6pd tomorrow +762 a10E76jX35YwquKCTA tomorrow +762 q5y2Vy1 tomorrow +6981 1FNNhmiFLGw425NA13g tomorrow +6981 K630vaVf tomorrow +6981 YdG61y00526u5 tomorrow +6981 a3EhVU6Wuy7ycJ7wY7h2gv tomorrow +6981 o4lvY20511w0EOX3P3I82p63 tomorrow diff --git ql/src/test/results/clientpositive/update_after_multiple_inserts.q.out ql/src/test/results/clientpositive/update_after_multiple_inserts.q.out new file mode 100644 index 0000000..cb8e319 --- /dev/null +++ ql/src/test/results/clientpositive/update_after_multiple_inserts.q.out @@ -0,0 +1,74 @@ +PREHOOK: query: create table acid_uami(i int, + de decimal(5,2), + vc varchar(128)) clustered by (i) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_uami +POSTHOOK: query: create table acid_uami(i int, + de decimal(5,2), + vc varchar(128)) clustered by (i) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_uami +PREHOOK: query: insert into table acid_uami values + (1, 109.23, 'mary had a little lamb'), + (6553, 923.19, 'its fleece was white as snow') +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@acid_uami +POSTHOOK: query: insert into table acid_uami values + (1, 109.23, 'mary had a little lamb'), + (6553, 923.19, 'its fleece was white as snow') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@acid_uami +POSTHOOK: Lineage: acid_uami.de EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: acid_uami.i EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: acid_uami.vc EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +PREHOOK: query: insert into table acid_uami values + (10, 119.23, 'and everywhere that mary went'), + (65530, 823.19, 'the lamb was sure to go') +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__2 +PREHOOK: Output: default@acid_uami +POSTHOOK: query: insert into table acid_uami values + (10, 119.23, 'and everywhere that mary went'), + (65530, 823.19, 'the lamb was sure to go') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__2 +POSTHOOK: Output: default@acid_uami +POSTHOOK: Lineage: acid_uami.de EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: acid_uami.i EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: acid_uami.vc EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +PREHOOK: query: select * from acid_uami order by de +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uami +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_uami order by de +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uami +#### A masked pattern was here #### +1 109.23 mary had a little lamb +10 119.23 and everywhere that mary went +65530 823.19 the lamb was sure to go +6553 923.19 its fleece was white as snow +PREHOOK: query: update acid_uami set de = 3.14 where de = 109.23 or de = 119.23 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uami +PREHOOK: Output: default@acid_uami +POSTHOOK: query: update acid_uami set de = 3.14 where de = 109.23 or de = 119.23 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uami +POSTHOOK: Output: default@acid_uami +PREHOOK: query: select * from acid_uami order by de +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uami +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_uami order by de +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uami +#### A masked pattern was here #### +1 3.14 mary had a little lamb +10 3.14 and everywhere that mary went +65530 823.19 the lamb was sure to go +6553 923.19 its fleece was white as snow diff --git ql/src/test/results/clientpositive/update_all_non_partitioned.q.out ql/src/test/results/clientpositive/update_all_non_partitioned.q.out new file mode 100644 index 0000000..fde6d8d --- /dev/null +++ ql/src/test/results/clientpositive/update_all_non_partitioned.q.out @@ -0,0 +1,62 @@ +PREHOOK: query: create table acid_uanp(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_uanp +POSTHOOK: query: create table acid_uanp(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_uanp +PREHOOK: query: insert into table acid_uanp select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint < 0 order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_uanp +POSTHOOK: query: insert into table acid_uanp select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint < 0 order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_uanp +POSTHOOK: Lineage: acid_uanp.a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_uanp.b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: select a,b from acid_uanp order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uanp +#### A masked pattern was here #### +POSTHOOK: query: select a,b from acid_uanp order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uanp +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa +-1073051226 A34p7oRr2WvUJNf +-1072910839 0iqrc5 +-1072081801 dPkN74F7 +-1072076362 2uLyD28144vklju213J1mr +-1071480828 aw724t8c5558x2xneC624 +-1071363017 Anj0oF +-1070883071 0ruyd6Y50JpdGRf6HqD +-1070551679 iUR3Q +-1069736047 k17Am8uPHWk02cEf1jet +PREHOOK: query: update acid_uanp set b = 'fred' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uanp +PREHOOK: Output: default@acid_uanp +POSTHOOK: query: update acid_uanp set b = 'fred' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uanp +POSTHOOK: Output: default@acid_uanp +PREHOOK: query: select a,b from acid_uanp order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uanp +#### A masked pattern was here #### +POSTHOOK: query: select a,b from acid_uanp order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uanp +#### A masked pattern was here #### +-1073279343 fred +-1073051226 fred +-1072910839 fred +-1072081801 fred +-1072076362 fred +-1071480828 fred +-1071363017 fred +-1070883071 fred +-1070551679 fred +-1069736047 fred diff --git ql/src/test/results/clientpositive/update_all_partitioned.q.out ql/src/test/results/clientpositive/update_all_partitioned.q.out new file mode 100644 index 0000000..3fae6a9 --- /dev/null +++ ql/src/test/results/clientpositive/update_all_partitioned.q.out @@ -0,0 +1,106 @@ +PREHOOK: query: create table acid_uap(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_uap +POSTHOOK: query: create table acid_uap(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_uap +PREHOOK: query: insert into table acid_uap partition (ds='today') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint < 0 order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_uap@ds=today +POSTHOOK: query: insert into table acid_uap partition (ds='today') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint < 0 order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_uap@ds=today +POSTHOOK: Lineage: acid_uap PARTITION(ds=today).a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_uap PARTITION(ds=today).b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: insert into table acid_uap partition (ds='tomorrow') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint > 10 order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_uap@ds=tomorrow +POSTHOOK: query: insert into table acid_uap partition (ds='tomorrow') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint > 10 order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_uap@ds=tomorrow +POSTHOOK: Lineage: acid_uap PARTITION(ds=tomorrow).a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_uap PARTITION(ds=tomorrow).b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: select a,b,ds from acid_uap order by a,b +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uap +PREHOOK: Input: default@acid_uap@ds=today +PREHOOK: Input: default@acid_uap@ds=tomorrow +#### A masked pattern was here #### +POSTHOOK: query: select a,b,ds from acid_uap order by a,b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uap +POSTHOOK: Input: default@acid_uap@ds=today +POSTHOOK: Input: default@acid_uap@ds=tomorrow +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa today +-1073051226 A34p7oRr2WvUJNf today +-1072910839 0iqrc5 today +-1072081801 dPkN74F7 today +-1072076362 2uLyD28144vklju213J1mr today +-1071480828 aw724t8c5558x2xneC624 today +-1071363017 Anj0oF today +-1070883071 0ruyd6Y50JpdGRf6HqD today +-1070551679 iUR3Q today +-1069736047 k17Am8uPHWk02cEf1jet today +762 3WsVeqb28VWEEOLI8ail tomorrow +762 40ks5556SV tomorrow +762 BLoMwUJ51ns6pd tomorrow +762 a10E76jX35YwquKCTA tomorrow +762 q5y2Vy1 tomorrow +6981 1FNNhmiFLGw425NA13g tomorrow +6981 K630vaVf tomorrow +6981 YdG61y00526u5 tomorrow +6981 a3EhVU6Wuy7ycJ7wY7h2gv tomorrow +6981 o4lvY20511w0EOX3P3I82p63 tomorrow +PREHOOK: query: update acid_uap set b = 'fred' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uap +PREHOOK: Input: default@acid_uap@ds=today +PREHOOK: Input: default@acid_uap@ds=tomorrow +PREHOOK: Output: default@acid_uap@ds=today +PREHOOK: Output: default@acid_uap@ds=tomorrow +POSTHOOK: query: update acid_uap set b = 'fred' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uap +POSTHOOK: Input: default@acid_uap@ds=today +POSTHOOK: Input: default@acid_uap@ds=tomorrow +POSTHOOK: Output: default@acid_uap@ds=today +POSTHOOK: Output: default@acid_uap@ds=tomorrow +PREHOOK: query: select a,b,ds from acid_uap order by a,b +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uap +PREHOOK: Input: default@acid_uap@ds=today +PREHOOK: Input: default@acid_uap@ds=tomorrow +#### A masked pattern was here #### +POSTHOOK: query: select a,b,ds from acid_uap order by a,b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uap +POSTHOOK: Input: default@acid_uap@ds=today +POSTHOOK: Input: default@acid_uap@ds=tomorrow +#### A masked pattern was here #### +-1073279343 fred today +-1073051226 fred today +-1072910839 fred today +-1072081801 fred today +-1072076362 fred today +-1071480828 fred today +-1071363017 fred today +-1070883071 fred today +-1070551679 fred today +-1069736047 fred today +762 fred tomorrow +762 fred tomorrow +762 fred tomorrow +762 fred tomorrow +762 fred tomorrow +6981 fred tomorrow +6981 fred tomorrow +6981 fred tomorrow +6981 fred tomorrow +6981 fred tomorrow diff --git ql/src/test/results/clientpositive/update_all_types.q.out ql/src/test/results/clientpositive/update_all_types.q.out new file mode 100644 index 0000000..36b4684 --- /dev/null +++ ql/src/test/results/clientpositive/update_all_types.q.out @@ -0,0 +1,153 @@ +PREHOOK: query: create table acid_uat(ti tinyint, + si smallint, + i int, + bi bigint, + f float, + d double, + de decimal(5,2), + t timestamp, + dt date, + s string, + vc varchar(128), + ch char(36), + b boolean) clustered by (i) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_uat +POSTHOOK: query: create table acid_uat(ti tinyint, + si smallint, + i int, + bi bigint, + f float, + d double, + de decimal(5,2), + t timestamp, + dt date, + s string, + vc varchar(128), + ch char(36), + b boolean) clustered by (i) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_uat +PREHOOK: query: insert into table acid_uat + select ctinyint, + csmallint, + cint, + cbigint, + cfloat, + cdouble, + cast(cfloat as decimal(5,2)), + ctimestamp1, + cast(ctimestamp2 as date), + cstring1, + cast(cstring1 as varchar(128)), + cast(cstring2 as char(36)), + cboolean1 + from alltypesorc where cint < 0 order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_uat +POSTHOOK: query: insert into table acid_uat + select ctinyint, + csmallint, + cint, + cbigint, + cfloat, + cdouble, + cast(cfloat as decimal(5,2)), + ctimestamp1, + cast(ctimestamp2 as date), + cstring1, + cast(cstring1 as varchar(128)), + cast(cstring2 as char(36)), + cboolean1 + from alltypesorc where cint < 0 order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_uat +POSTHOOK: Lineage: acid_uat.b SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean1, type:boolean, comment:null), ] +POSTHOOK: Lineage: acid_uat.bi SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ] +POSTHOOK: Lineage: acid_uat.ch EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] +POSTHOOK: Lineage: acid_uat.d SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: acid_uat.de EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: acid_uat.dt EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] +POSTHOOK: Lineage: acid_uat.f SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: acid_uat.i SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_uat.s SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: acid_uat.si SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] +POSTHOOK: Lineage: acid_uat.t SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] +POSTHOOK: Lineage: acid_uat.ti SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +POSTHOOK: Lineage: acid_uat.vc EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: select * from acid_uat order by i +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uat +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_uat order by i +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uat +#### A masked pattern was here #### +11 NULL -1073279343 -1595604468 11.0 NULL 11.0 1969-12-31 16:00:02.351 NULL oj1YrV5Wa oj1YrV5Wa P76636jJ6qM17d7DIy true +NULL -7382 -1073051226 -1887561756 NULL -7382.0 NULL NULL 1970-01-01 A34p7oRr2WvUJNf A34p7oRr2WvUJNf 4hA4KQj2vD3fI6gX82220d false +11 NULL -1072910839 2048385991 11.0 NULL 11.0 1969-12-31 16:00:02.351 NULL 0iqrc5 0iqrc5 KbaDXiN85adbHRx58v false +NULL 8373 -1072081801 1864027286 NULL 8373.0 NULL NULL 1970-01-01 dPkN74F7 dPkN74F7 4KWs6gw7lv2WYd66P true +NULL -5470 -1072076362 1864027286 NULL -5470.0 NULL NULL 1970-01-01 2uLyD28144vklju213J1mr 2uLyD28144vklju213J1mr 4KWs6gw7lv2WYd66P true +-51 NULL -1071480828 -1401575336 -51.0 NULL -51.0 1969-12-31 16:00:08.451 NULL aw724t8c5558x2xneC624 aw724t8c5558x2xneC624 4uE7l74tESBiKfu7c8wM7GA true +8 NULL -1071363017 1349676361 8.0 NULL 8.0 1969-12-31 16:00:15.892 NULL Anj0oF Anj0oF IwE1G7Qb0B1NEfV030g true +NULL -741 -1070883071 -1645852809 NULL -741.0 NULL NULL 1970-01-01 0ruyd6Y50JpdGRf6HqD 0ruyd6Y50JpdGRf6HqD xH7445Rals48VOulSyR5F false +NULL -947 -1070551679 1864027286 NULL -947.0 NULL NULL 1970-01-01 iUR3Q iUR3Q 4KWs6gw7lv2WYd66P false +11 NULL -1069736047 -453772520 11.0 NULL 11.0 1969-12-31 16:00:02.351 NULL k17Am8uPHWk02cEf1jet k17Am8uPHWk02cEf1jet qrXLLNX1 true +PREHOOK: query: update acid_uat set + ti = 1, + si = 2, + i = 3, + bi = 4, + f = 3.14, + d = 6.28, + de = 5.99, + t = '2014-09-01 09:44.23.23', + dt = '2014-09-01', + s = 'its a beautiful day in the neighbhorhood', + vc = 'a beautiful day for a neighbor', + ch = 'wont you be mine', + b = true + where s = '0ruyd6Y50JpdGRf6HqD' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uat +PREHOOK: Output: default@acid_uat +POSTHOOK: query: update acid_uat set + ti = 1, + si = 2, + i = 3, + bi = 4, + f = 3.14, + d = 6.28, + de = 5.99, + t = '2014-09-01 09:44.23.23', + dt = '2014-09-01', + s = 'its a beautiful day in the neighbhorhood', + vc = 'a beautiful day for a neighbor', + ch = 'wont you be mine', + b = true + where s = '0ruyd6Y50JpdGRf6HqD' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uat +POSTHOOK: Output: default@acid_uat +PREHOOK: query: select * from acid_uat order by i +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uat +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_uat order by i +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uat +#### A masked pattern was here #### +11 NULL -1073279343 -1595604468 11.0 NULL 11.0 1969-12-31 16:00:02.351 NULL oj1YrV5Wa oj1YrV5Wa P76636jJ6qM17d7DIy true +NULL -7382 -1073051226 -1887561756 NULL -7382.0 NULL NULL 1970-01-01 A34p7oRr2WvUJNf A34p7oRr2WvUJNf 4hA4KQj2vD3fI6gX82220d false +11 NULL -1072910839 2048385991 11.0 NULL 11.0 1969-12-31 16:00:02.351 NULL 0iqrc5 0iqrc5 KbaDXiN85adbHRx58v false +NULL 8373 -1072081801 1864027286 NULL 8373.0 NULL NULL 1970-01-01 dPkN74F7 dPkN74F7 4KWs6gw7lv2WYd66P true +NULL -5470 -1072076362 1864027286 NULL -5470.0 NULL NULL 1970-01-01 2uLyD28144vklju213J1mr 2uLyD28144vklju213J1mr 4KWs6gw7lv2WYd66P true +-51 NULL -1071480828 -1401575336 -51.0 NULL -51.0 1969-12-31 16:00:08.451 NULL aw724t8c5558x2xneC624 aw724t8c5558x2xneC624 4uE7l74tESBiKfu7c8wM7GA true +8 NULL -1071363017 1349676361 8.0 NULL 8.0 1969-12-31 16:00:15.892 NULL Anj0oF Anj0oF IwE1G7Qb0B1NEfV030g true +NULL -947 -1070551679 1864027286 NULL -947.0 NULL NULL 1970-01-01 iUR3Q iUR3Q 4KWs6gw7lv2WYd66P false +11 NULL -1069736047 -453772520 11.0 NULL 11.0 1969-12-31 16:00:02.351 NULL k17Am8uPHWk02cEf1jet k17Am8uPHWk02cEf1jet qrXLLNX1 true +1 2 3 4 3.14 6.28 5.99 NULL 2014-09-01 its a beautiful day in the neighbhorhood a beautiful day for a neighbor wont you be mine true diff --git ql/src/test/results/clientpositive/update_orig_table.q.out ql/src/test/results/clientpositive/update_orig_table.q.out new file mode 100644 index 0000000..de7dc86 --- /dev/null +++ ql/src/test/results/clientpositive/update_orig_table.q.out @@ -0,0 +1,59 @@ +PREHOOK: query: create table acid_uot( + ctinyint TINYINT, + csmallint SMALLINT, + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE, + cstring1 STRING, + cstring2 STRING, + ctimestamp1 TIMESTAMP, + ctimestamp2 TIMESTAMP, + cboolean1 BOOLEAN, +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_uot +POSTHOOK: query: create table acid_uot( + ctinyint TINYINT, + csmallint SMALLINT, + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE, + cstring1 STRING, + cstring2 STRING, + ctimestamp1 TIMESTAMP, + ctimestamp2 TIMESTAMP, + cboolean1 BOOLEAN, +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_uot +PREHOOK: query: update acid_uot set cstring1 = 'fred' where cint < -1070551679 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uot +PREHOOK: Output: default@acid_uot +POSTHOOK: query: update acid_uot set cstring1 = 'fred' where cint < -1070551679 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uot +POSTHOOK: Output: default@acid_uot +PREHOOK: query: select * from acid_uot where cstring1 = 'fred' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uot +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_uot where cstring1 = 'fred' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uot +#### A masked pattern was here #### +-51 NULL -1071480828 -1401575336 -51.0 NULL fred 4uE7l74tESBiKfu7c8wM7GA 1969-12-31 16:00:08.451 NULL true true +NULL -5470 -1072076362 1864027286 NULL -5470.0 fred 4KWs6gw7lv2WYd66P NULL 1969-12-31 16:00:01.836 true true +NULL 8373 -1072081801 1864027286 NULL 8373.0 fred 4KWs6gw7lv2WYd66P NULL 1969-12-31 15:59:56.465 true true +8 NULL -1071363017 1349676361 8.0 NULL fred IwE1G7Qb0B1NEfV030g 1969-12-31 16:00:15.892 NULL true true +NULL -7382 -1073051226 -1887561756 NULL -7382.0 fred 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:10.331 false false +11 NULL -1073279343 -1595604468 11.0 NULL fred P76636jJ6qM17d7DIy 1969-12-31 16:00:02.351 NULL true true +11 NULL -1072910839 2048385991 11.0 NULL fred KbaDXiN85adbHRx58v 1969-12-31 16:00:02.351 NULL false false +NULL -741 -1070883071 -1645852809 NULL -741.0 fred xH7445Rals48VOulSyR5F NULL 1969-12-31 15:59:51.293 false false +#### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/update_tmp_table.q.out ql/src/test/results/clientpositive/update_tmp_table.q.out new file mode 100644 index 0000000..8180f06 --- /dev/null +++ ql/src/test/results/clientpositive/update_tmp_table.q.out @@ -0,0 +1,62 @@ +PREHOOK: query: create table acid_utt(a int, b varchar(128)) clustered by (b) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_utt +POSTHOOK: query: create table acid_utt(a int, b varchar(128)) clustered by (b) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_utt +PREHOOK: query: insert into table acid_utt select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_utt +POSTHOOK: query: insert into table acid_utt select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_utt +POSTHOOK: Lineage: acid_utt.a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_utt.b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: select a,b from acid_utt order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_utt +#### A masked pattern was here #### +POSTHOOK: query: select a,b from acid_utt order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_utt +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa +-1073051226 A34p7oRr2WvUJNf +-1072910839 0iqrc5 +-1072081801 dPkN74F7 +-1072076362 2uLyD28144vklju213J1mr +-1071480828 aw724t8c5558x2xneC624 +-1071363017 Anj0oF +-1070883071 0ruyd6Y50JpdGRf6HqD +-1070551679 iUR3Q +-1069736047 k17Am8uPHWk02cEf1jet +PREHOOK: query: update acid_utt set b = 'fred' where b = '0ruyd6Y50JpdGRf6HqD' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_utt +PREHOOK: Output: default@acid_utt +POSTHOOK: query: update acid_utt set b = 'fred' where b = '0ruyd6Y50JpdGRf6HqD' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_utt +POSTHOOK: Output: default@acid_utt +PREHOOK: query: select * from acid_utt order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_utt +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_utt order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_utt +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa +-1073051226 A34p7oRr2WvUJNf +-1072910839 0iqrc5 +-1072081801 dPkN74F7 +-1072076362 2uLyD28144vklju213J1mr +-1071480828 aw724t8c5558x2xneC624 +-1071363017 Anj0oF +-1070883071 fred +-1070551679 iUR3Q +-1069736047 k17Am8uPHWk02cEf1jet diff --git ql/src/test/results/clientpositive/update_two_cols.q.out ql/src/test/results/clientpositive/update_two_cols.q.out new file mode 100644 index 0000000..553608f --- /dev/null +++ ql/src/test/results/clientpositive/update_two_cols.q.out @@ -0,0 +1,63 @@ +PREHOOK: query: create table acid_utc(a int, b varchar(128), c float) clustered by (a) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_utc +POSTHOOK: query: create table acid_utc(a int, b varchar(128), c float) clustered by (a) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_utc +PREHOOK: query: insert into table acid_utc select cint, cast(cstring1 as varchar(128)), cfloat from alltypesorc where cint < 0 order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_utc +POSTHOOK: query: insert into table acid_utc select cint, cast(cstring1 as varchar(128)), cfloat from alltypesorc where cint < 0 order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_utc +POSTHOOK: Lineage: acid_utc.a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_utc.b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: acid_utc.c SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] +PREHOOK: query: select * from acid_utc order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_utc +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_utc order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_utc +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa 11.0 +-1073051226 A34p7oRr2WvUJNf NULL +-1072910839 0iqrc5 11.0 +-1072081801 dPkN74F7 NULL +-1072076362 2uLyD28144vklju213J1mr NULL +-1071480828 aw724t8c5558x2xneC624 -51.0 +-1071363017 Anj0oF 8.0 +-1070883071 0ruyd6Y50JpdGRf6HqD NULL +-1070551679 iUR3Q NULL +-1069736047 k17Am8uPHWk02cEf1jet 11.0 +PREHOOK: query: update acid_utc set b = 'fred',c = 3.14 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_utc +PREHOOK: Output: default@acid_utc +POSTHOOK: query: update acid_utc set b = 'fred',c = 3.14 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_utc +POSTHOOK: Output: default@acid_utc +PREHOOK: query: select * from acid_utc order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_utc +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_utc order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_utc +#### A masked pattern was here #### +-1073279343 fred 3.14 +-1073051226 fred 3.14 +-1072910839 fred 3.14 +-1072081801 fred 3.14 +-1072076362 fred 3.14 +-1071480828 fred 3.14 +-1071363017 fred 3.14 +-1070883071 fred 3.14 +-1070551679 fred 3.14 +-1069736047 fred 3.14 diff --git ql/src/test/results/clientpositive/update_where_no_match.q.out ql/src/test/results/clientpositive/update_where_no_match.q.out new file mode 100644 index 0000000..afef267 --- /dev/null +++ ql/src/test/results/clientpositive/update_where_no_match.q.out @@ -0,0 +1,62 @@ +PREHOOK: query: create table acid_wnm(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_wnm +POSTHOOK: query: create table acid_wnm(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_wnm +PREHOOK: query: insert into table acid_wnm select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_wnm +POSTHOOK: query: insert into table acid_wnm select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_wnm +POSTHOOK: Lineage: acid_wnm.a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_wnm.b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: select a,b from acid_wnm order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_wnm +#### A masked pattern was here #### +POSTHOOK: query: select a,b from acid_wnm order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_wnm +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa +-1073051226 A34p7oRr2WvUJNf +-1072910839 0iqrc5 +-1072081801 dPkN74F7 +-1072076362 2uLyD28144vklju213J1mr +-1071480828 aw724t8c5558x2xneC624 +-1071363017 Anj0oF +-1070883071 0ruyd6Y50JpdGRf6HqD +-1070551679 iUR3Q +-1069736047 k17Am8uPHWk02cEf1jet +PREHOOK: query: update acid_wnm set b = 'fred' where b = 'nosuchvalue' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_wnm +PREHOOK: Output: default@acid_wnm +POSTHOOK: query: update acid_wnm set b = 'fred' where b = 'nosuchvalue' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_wnm +POSTHOOK: Output: default@acid_wnm +PREHOOK: query: select * from acid_wnm order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_wnm +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_wnm order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_wnm +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa +-1073051226 A34p7oRr2WvUJNf +-1072910839 0iqrc5 +-1072081801 dPkN74F7 +-1072076362 2uLyD28144vklju213J1mr +-1071480828 aw724t8c5558x2xneC624 +-1071363017 Anj0oF +-1070883071 0ruyd6Y50JpdGRf6HqD +-1070551679 iUR3Q +-1069736047 k17Am8uPHWk02cEf1jet diff --git ql/src/test/results/clientpositive/update_where_non_partitioned.q.out ql/src/test/results/clientpositive/update_where_non_partitioned.q.out new file mode 100644 index 0000000..5c79379 --- /dev/null +++ ql/src/test/results/clientpositive/update_where_non_partitioned.q.out @@ -0,0 +1,62 @@ +PREHOOK: query: create table acid_uwnp(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_uwnp +POSTHOOK: query: create table acid_uwnp(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_uwnp +PREHOOK: query: insert into table acid_uwnp select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_uwnp +POSTHOOK: query: insert into table acid_uwnp select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_uwnp +POSTHOOK: Lineage: acid_uwnp.a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_uwnp.b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: select a,b from acid_uwnp order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uwnp +#### A masked pattern was here #### +POSTHOOK: query: select a,b from acid_uwnp order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uwnp +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa +-1073051226 A34p7oRr2WvUJNf +-1072910839 0iqrc5 +-1072081801 dPkN74F7 +-1072076362 2uLyD28144vklju213J1mr +-1071480828 aw724t8c5558x2xneC624 +-1071363017 Anj0oF +-1070883071 0ruyd6Y50JpdGRf6HqD +-1070551679 iUR3Q +-1069736047 k17Am8uPHWk02cEf1jet +PREHOOK: query: update acid_uwnp set b = 'fred' where b = '0ruyd6Y50JpdGRf6HqD' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uwnp +PREHOOK: Output: default@acid_uwnp +POSTHOOK: query: update acid_uwnp set b = 'fred' where b = '0ruyd6Y50JpdGRf6HqD' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uwnp +POSTHOOK: Output: default@acid_uwnp +PREHOOK: query: select * from acid_uwnp order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uwnp +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_uwnp order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uwnp +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa +-1073051226 A34p7oRr2WvUJNf +-1072910839 0iqrc5 +-1072081801 dPkN74F7 +-1072076362 2uLyD28144vklju213J1mr +-1071480828 aw724t8c5558x2xneC624 +-1071363017 Anj0oF +-1070883071 fred +-1070551679 iUR3Q +-1069736047 k17Am8uPHWk02cEf1jet diff --git ql/src/test/results/clientpositive/update_where_partitioned.q.out ql/src/test/results/clientpositive/update_where_partitioned.q.out new file mode 100644 index 0000000..b83c52a --- /dev/null +++ ql/src/test/results/clientpositive/update_where_partitioned.q.out @@ -0,0 +1,106 @@ +PREHOOK: query: create table acid_uwp(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_uwp +POSTHOOK: query: create table acid_uwp(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_uwp +PREHOOK: query: insert into table acid_uwp partition (ds='today') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint < 0 order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_uwp@ds=today +POSTHOOK: query: insert into table acid_uwp partition (ds='today') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint < 0 order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_uwp@ds=today +POSTHOOK: Lineage: acid_uwp PARTITION(ds=today).a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_uwp PARTITION(ds=today).b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: insert into table acid_uwp partition (ds='tomorrow') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint > 100 order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_uwp@ds=tomorrow +POSTHOOK: query: insert into table acid_uwp partition (ds='tomorrow') select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint is not null and cint > 100 order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_uwp@ds=tomorrow +POSTHOOK: Lineage: acid_uwp PARTITION(ds=tomorrow).a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_uwp PARTITION(ds=tomorrow).b EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: select a,b,ds from acid_uwp order by a, ds, b +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uwp +PREHOOK: Input: default@acid_uwp@ds=today +PREHOOK: Input: default@acid_uwp@ds=tomorrow +#### A masked pattern was here #### +POSTHOOK: query: select a,b,ds from acid_uwp order by a, ds, b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uwp +POSTHOOK: Input: default@acid_uwp@ds=today +POSTHOOK: Input: default@acid_uwp@ds=tomorrow +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa today +-1073051226 A34p7oRr2WvUJNf today +-1072910839 0iqrc5 today +-1072081801 dPkN74F7 today +-1072076362 2uLyD28144vklju213J1mr today +-1071480828 aw724t8c5558x2xneC624 today +-1071363017 Anj0oF today +-1070883071 0ruyd6Y50JpdGRf6HqD today +-1070551679 iUR3Q today +-1069736047 k17Am8uPHWk02cEf1jet today +762 3WsVeqb28VWEEOLI8ail tomorrow +762 40ks5556SV tomorrow +762 BLoMwUJ51ns6pd tomorrow +762 a10E76jX35YwquKCTA tomorrow +762 q5y2Vy1 tomorrow +6981 1FNNhmiFLGw425NA13g tomorrow +6981 K630vaVf tomorrow +6981 YdG61y00526u5 tomorrow +6981 a3EhVU6Wuy7ycJ7wY7h2gv tomorrow +6981 o4lvY20511w0EOX3P3I82p63 tomorrow +PREHOOK: query: update acid_uwp set b = 'fred' where b = 'k17Am8uPHWk02cEf1jet' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uwp +PREHOOK: Input: default@acid_uwp@ds=today +PREHOOK: Input: default@acid_uwp@ds=tomorrow +PREHOOK: Output: default@acid_uwp@ds=today +PREHOOK: Output: default@acid_uwp@ds=tomorrow +POSTHOOK: query: update acid_uwp set b = 'fred' where b = 'k17Am8uPHWk02cEf1jet' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uwp +POSTHOOK: Input: default@acid_uwp@ds=today +POSTHOOK: Input: default@acid_uwp@ds=tomorrow +POSTHOOK: Output: default@acid_uwp@ds=today +POSTHOOK: Output: default@acid_uwp@ds=tomorrow +PREHOOK: query: select * from acid_uwp order by a, ds, b +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_uwp +PREHOOK: Input: default@acid_uwp@ds=today +PREHOOK: Input: default@acid_uwp@ds=tomorrow +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_uwp order by a, ds, b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_uwp +POSTHOOK: Input: default@acid_uwp@ds=today +POSTHOOK: Input: default@acid_uwp@ds=tomorrow +#### A masked pattern was here #### +-1073279343 oj1YrV5Wa today +-1073051226 A34p7oRr2WvUJNf today +-1072910839 0iqrc5 today +-1072081801 dPkN74F7 today +-1072076362 2uLyD28144vklju213J1mr today +-1071480828 aw724t8c5558x2xneC624 today +-1071363017 Anj0oF today +-1070883071 0ruyd6Y50JpdGRf6HqD today +-1070551679 iUR3Q today +-1069736047 fred today +762 3WsVeqb28VWEEOLI8ail tomorrow +762 40ks5556SV tomorrow +762 BLoMwUJ51ns6pd tomorrow +762 a10E76jX35YwquKCTA tomorrow +762 q5y2Vy1 tomorrow +6981 1FNNhmiFLGw425NA13g tomorrow +6981 K630vaVf tomorrow +6981 YdG61y00526u5 tomorrow +6981 a3EhVU6Wuy7ycJ7wY7h2gv tomorrow +6981 o4lvY20511w0EOX3P3I82p63 tomorrow