diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 711dfbdc1f..225a5b3598 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2982,6 +2982,9 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal + "When it is set to false, only [a-zA-Z_0-9]+ are supported.\n" + "The only supported special character right now is '/'. This flag applies only to quoted table names.\n" + "The default value is true."), + HIVE_CREATE_TABLES_AS_ACID("hive.create.as.acid", true, + "Whether the eligible tables should be created as full ACID by default. Does \n" + + "not apply to external tables, the ones using storage handlers, etc."), HIVE_CREATE_TABLES_AS_INSERT_ONLY("hive.create.as.insert.only", false, "Whether the eligible tables should be created as ACID insert-only by default. Does \n" + "not apply to external tables, the ones using storage handlers, etc."), diff --git ql/src/java/org/apache/hadoop/hive/ql/Driver.java ql/src/java/org/apache/hadoop/hive/ql/Driver.java index b168906b44..a793313465 100644 --- ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -1332,6 +1332,9 @@ public void releaseLocksAndCommitOrRollback(boolean commit, HiveTxnManager txnMa // If we've opened a transaction we need to commit or rollback rather than explicitly // releasing the locks. conf.unset(ValidTxnList.VALID_TXNS_KEY); + if(!checkConcurrency()) { + return; + } if (txnMgr.isTxnOpen()) { if (commit) { if(conf.getBoolVar(ConfVars.HIVE_IN_TEST) && conf.getBoolVar(ConfVars.HIVETESTMODEROLLBACKTXN)) { diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index dcda8b3e00..8d65643319 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -108,6 +108,7 @@ import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.hooks.WriteEntity; +import org.apache.hadoop.hive.ql.io.AcidInputFormat; import org.apache.hadoop.hive.ql.io.AcidOutputFormat; import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.io.AcidUtils.Operation; @@ -119,6 +120,7 @@ import org.apache.hadoop.hive.ql.lib.Dispatcher; import org.apache.hadoop.hive.ql.lib.GraphWalker; import org.apache.hadoop.hive.ql.lib.Node; +import org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; import org.apache.hadoop.hive.ql.metadata.DummyPartition; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -12212,14 +12214,6 @@ private void validate(Task task, boolean reworkMapredWor validate(childTask, reworkMapredWork); } } - - /** - * Get the row resolver given an operator. - */ - public RowResolver getRowResolver(Operator opt) { - return opParseCtx.get(opt).getRowResolver(); - } - /** * Add default properties for table property. If a default parameter exists * in the tblProp, the value in tblProp will be kept. @@ -12229,7 +12223,8 @@ public RowResolver getRowResolver(Operator opt) { * @return Modified table property map */ private Map addDefaultProperties( - Map tblProp, boolean isExt, StorageFormat storageFormat) { + Map tblProp, boolean isExt, StorageFormat storageFormat, + String qualifiedTableName, List sortCols) { Map retValue; if (tblProp == null) { retValue = new HashMap(); @@ -12248,16 +12243,45 @@ public RowResolver getRowResolver(Operator opt) { } } } - if (HiveConf.getBoolVar(conf, ConfVars.HIVE_CREATE_TABLES_AS_INSERT_ONLY) + boolean makeInsertOnly = HiveConf.getBoolVar(conf, ConfVars.HIVE_CREATE_TABLES_AS_INSERT_ONLY); + boolean makeAcid = HiveConf.getBoolVar(conf, ConfVars.HIVE_CREATE_TABLES_AS_ACID) && + HiveConf.getBoolVar(conf, ConfVars.HIVE_SUPPORT_CONCURRENCY) && + DbTxnManager.class.getCanonicalName().equals(HiveConf.getVar(conf, ConfVars.HIVE_TXN_MANAGER)); + if ((makeInsertOnly || makeAcid) && !isExt && StringUtils.isBlank(storageFormat.getStorageHandler()) + //don't overwrite user choice if transactional attribute is explicitly set && !retValue.containsKey(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL)) { - retValue.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, "true"); - String oldProps = retValue.get(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES); - if (oldProps != null) { - LOG.warn("Non-transactional table has transactional properties; overwriting " + oldProps); + if(makeInsertOnly) { + retValue.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, "true"); + retValue.put(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES, + TransactionalValidationListener.INSERTONLY_TRANSACTIONAL_PROPERTY); + } + if(makeAcid) { + /*for CTAS, TransactionalValidationListener.makeAcid() runs to late to make table Acid + so the initial write ends up running as non-acid...*/ + try { + Class inputFormatClass = storageFormat.getInputFormat() == null ? null : + Class.forName(storageFormat.getInputFormat()); + Class outputFormatClass = storageFormat.getOutputFormat() == null ? null : + Class.forName(storageFormat.getOutputFormat()); + if (inputFormatClass == null || outputFormatClass == null || + !AcidInputFormat.class.isAssignableFrom(inputFormatClass) || + !AcidOutputFormat.class.isAssignableFrom(outputFormatClass)) { + return retValue; + } + } catch (ClassNotFoundException e) { + LOG.warn("Could not verify InputFormat=" + storageFormat.getInputFormat() + " or OutputFormat=" + + storageFormat.getOutputFormat() + " for " + qualifiedTableName); + return retValue; + } + if(sortCols != null && !sortCols.isEmpty()) { + return retValue; + } + retValue.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, "true"); + retValue.put(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES, + TransactionalValidationListener.DEFAULT_TRANSACTIONAL_PROPERTY); + LOG.info("Automatically chose to make " + qualifiedTableName + " acid."); } - retValue.put(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES, - TransactionalValidationListener.INSERTONLY_TRANSACTIONAL_PROPERTY); } return retValue; } @@ -12482,7 +12506,7 @@ ASTNode analyzeCreateTable( switch (command_type) { case CREATE_TABLE: // REGULAR CREATE TABLE DDL - tblProps = addDefaultProperties(tblProps, isExt, storageFormat); + tblProps = addDefaultProperties(tblProps, isExt, storageFormat, dbDotTab, sortCols); CreateTableDesc crtTblDesc = new CreateTableDesc(dbDotTab, isExt, isTemporary, cols, partCols, bucketCols, sortCols, numBuckets, rowFormatParams.fieldDelim, @@ -12503,7 +12527,7 @@ ASTNode analyzeCreateTable( break; case CTLT: // create table like - tblProps = addDefaultProperties(tblProps, isExt, storageFormat); + tblProps = addDefaultProperties(tblProps, isExt, storageFormat, dbDotTab, sortCols); if (isTemporary) { Table likeTable = getTable(likeTableName, false); @@ -12580,7 +12604,7 @@ ASTNode analyzeCreateTable( } } - tblProps = addDefaultProperties(tblProps, isExt, storageFormat); + tblProps = addDefaultProperties(tblProps, isExt, storageFormat, dbDotTab, sortCols); tableDesc = new CreateTableDesc(qualifiedTabName[0], dbDotTab, isExt, isTemporary, cols, partCols, bucketCols, sortCols, numBuckets, rowFormatParams.fieldDelim, rowFormatParams.fieldEscape, rowFormatParams.collItemDelim, rowFormatParams.mapKeyDelim, diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java index da1031300a..5eda505912 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.metastore; import java.io.IOException; +import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -34,6 +35,7 @@ import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.events.PreAlterTableEvent; import org.apache.hadoop.hive.metastore.events.PreCreateTableEvent; import org.apache.hadoop.hive.metastore.events.PreEventContext; @@ -185,6 +187,57 @@ private void handleAlterTableTransactionalProp(PreAlterTableEvent context) throw } /** + * Want to make a a newly create table Acid (unless it explicitly has transactional=false param) + * if table can support it. Also see SemanticAnalyzer.addDefaultProperties() which performs the + * same logic. This code path is more general since it is activated even if you create a table + * via Thrift, WebHCat etc but some operations like CTAS create the table (metastore object) as + * the last step (i.e. after the data is written) but write itself is has to be aware of the type + * of table so this Listener is too late. + */ + private void makeAcid(Table newTable) throws MetaException { + if(newTable.getParameters() != null && + newTable.getParameters().containsKey(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL)) { + LOG.info("Could not make " + Warehouse.getQualifiedName(newTable) + " acid: already has " + + hive_metastoreConstants.TABLE_IS_TRANSACTIONAL + "=" + + newTable.getParameters().get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL)); + return; + } + Configuration conf = MetastoreConf.newMetastoreConf(); + boolean makeAcid = + //no point making an acid table if these other props are not set since it will just throw + //exceptions when someone tries to use the table. + MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.CREATE_TABLES_AS_ACID) && + MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.HIVE_SUPPORT_CONCURRENCY) && + "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager".equals( + MetastoreConf.getVar(conf, MetastoreConf.ConfVars.HIVE_TXN_MANAGER) + ); + + if(makeAcid) { + if(!conformToAcid(newTable)) { + LOG.info("Could not make " + Warehouse.getQualifiedName(newTable) + " acid: wrong IO format"); + return; + } + if(!TableType.MANAGED_TABLE.toString().equalsIgnoreCase(newTable.getTableType())) { + //todo should this check be in conformToAcid()? + LOG.info("Could not make " + Warehouse.getQualifiedName(newTable) + " acid: it's " + + newTable.getTableType()); + return; + } + if(newTable.getSd().getSortColsSize() > 0) { + LOG.info("Could not make " + Warehouse.getQualifiedName(newTable) + " acid: it's sorted"); + return; + } + //check if orc and not sorted + Map parameters = newTable.getParameters(); + if (parameters == null || parameters.isEmpty()) { + parameters = new HashMap<>(); + } + parameters.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, "true"); + newTable.setParameters(parameters); + LOG.info("Automatically chose to make " + Warehouse.getQualifiedName(newTable) + " acid."); + } + } + /** * Normalize case and make sure: * 1. 'true' is the only value to be set for 'transactional' (if set at all) * 2. If set to 'true', we should also enforce bucketing and ORC format @@ -193,6 +246,7 @@ private void handleCreateTableTransactionalProp(PreCreateTableEvent context) thr Table newTable = context.getTable(); Map parameters = newTable.getParameters(); if (parameters == null || parameters.isEmpty()) { + makeAcid(newTable); return; } String transactional = null; @@ -212,6 +266,7 @@ private void handleCreateTableTransactionalProp(PreCreateTableEvent context) thr } if (transactional == null) { + makeAcid(newTable); return; } @@ -262,11 +317,13 @@ private void normazlieTransactionalPropertyDefault(Table table) { * Check that InputFormatClass/OutputFormatClass should implement * AcidInputFormat/AcidOutputFormat */ - private boolean conformToAcid(Table table) throws MetaException { + private static boolean conformToAcid(Table table) throws MetaException { StorageDescriptor sd = table.getSd(); try { - Class inputFormatClass = Class.forName(sd.getInputFormat()); - Class outputFormatClass = Class.forName(sd.getOutputFormat()); + Class inputFormatClass = sd.getInputFormat() == null ? null : + Class.forName(sd.getInputFormat()); + Class outputFormatClass = sd.getOutputFormat() == null ? null : + Class.forName(sd.getOutputFormat()); if (inputFormatClass == null || outputFormatClass == null || !Class.forName("org.apache.hadoop.hive.ql.io.AcidInputFormat").isAssignableFrom(inputFormatClass) || @@ -274,7 +331,9 @@ private boolean conformToAcid(Table table) throws MetaException { return false; } } catch (ClassNotFoundException e) { - throw new MetaException("Invalid input/output format for table"); + LOG.warn("Could not verify InputFormat=" + sd.getInputFormat() + " or OutputFormat=" + + sd.getOutputFormat() + " for " + Warehouse.getQualifiedName(table)); + return false; } return true; diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index b46cc38a22..9bd3d6954f 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -53,7 +53,7 @@ /** * A set of definitions of config values used by the Metastore. One of the key aims of this * class is to provide backwards compatibility with existing Hive configuration keys while - * allowing the metastore to have its own, Hive independant keys. For this reason access to the + * allowing the metastore to have its own, Hive independent keys. For this reason access to the * underlying Configuration object should always be done via the static methods provided here * rather than directly via {@link Configuration#get(String)} and * {@link Configuration#set(String, String)}. All the methods of this class will handle checking @@ -373,6 +373,9 @@ public static ConfVars getMetaConf(String name) { CONNECTION_USER_NAME("javax.jdo.option.ConnectionUserName", "javax.jdo.option.ConnectionUserName", "APP", "Username to use against metastore database"), + CREATE_TABLES_AS_ACID("metastore.create.as.acid", "hive.create.as.acid", true, + "Whether the eligible tables should be created as full ACID by default. Does \n" + + "not apply to external tables, the ones using storage handlers, etc."), COUNT_OPEN_TXNS_INTERVAL("metastore.count.open.txns.interval", "hive.count.open.txns.interval", 1, TimeUnit.SECONDS, "Time in seconds between checks to count open transactions."), DATANUCLEUS_AUTOSTART("datanucleus.autoStartMechanismMode",