diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index be83489cb3..44d710e852 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -531,7 +531,7 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "The smaller it is the more load there will be on the jobtracker, the higher it is the less granular the caught will be."), DYNAMICPARTITIONING("hive.exec.dynamic.partition", true, "Whether or not to allow dynamic partitions in DML/DDL."), - DYNAMICPARTITIONINGMODE("hive.exec.dynamic.partition.mode", "strict", + DYNAMICPARTITIONINGMODE("hive.exec.dynamic.partition.mode", "nostrict", "In strict mode, the user must specify at least one static partition\n" + "in case the user accidentally overwrites all partitions.\n" + "In nonstrict mode all partitions are allowed to be dynamic."), @@ -1845,7 +1845,7 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "filter operators."), // Concurrency - HIVE_SUPPORT_CONCURRENCY("hive.support.concurrency", false, + HIVE_SUPPORT_CONCURRENCY("hive.support.concurrency", !false, "Whether Hive supports concurrency control or not. \n" + "A ZooKeeper instance must be up and running when using zookeeper Hive lock manager "), HIVE_LOCK_MANAGER("hive.lock.manager", "org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager", ""), @@ -1894,7 +1894,7 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal // Transactions HIVE_TXN_MANAGER("hive.txn.manager", - "org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager", + "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager", "Set to org.apache.hadoop.hive.ql.lockmgr.DbTxnManager as part of turning on Hive\n" + "transactions, which also requires appropriate settings for hive.compactor.initiator.on,\n" + "hive.compactor.worker.threads, hive.support.concurrency (true),\n" + @@ -2982,6 +2982,9 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal + "When it is set to false, only [a-zA-Z_0-9]+ are supported.\n" + "The only supported special character right now is '/'. This flag applies only to quoted table names.\n" + "The default value is true."), + HIVE_CREATE_TABLES_AS_ACID("hive.create.as.acid", false, + "Whether the eligible tables should be created as full ACID by default. Does \n" + + "not apply to external tables, the ones using storage handlers, etc."), HIVE_CREATE_TABLES_AS_INSERT_ONLY("hive.create.as.insert.only", false, "Whether the eligible tables should be created as ACID insert-only by default. Does \n" + "not apply to external tables, the ones using storage handlers, etc."), diff --git ql/src/java/org/apache/hadoop/hive/ql/Driver.java ql/src/java/org/apache/hadoop/hive/ql/Driver.java index d632ee80c3..cfa38f40cb 100644 --- ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -1337,6 +1337,9 @@ public void releaseLocksAndCommitOrRollback(boolean commit, HiveTxnManager txnMa // If we've opened a transaction we need to commit or rollback rather than explicitly // releasing the locks. conf.unset(ValidTxnList.VALID_TXNS_KEY); + if(!checkConcurrency()) { + return; + } if (txnMgr.isTxnOpen()) { if (commit) { if(conf.getBoolVar(ConfVars.HIVE_IN_TEST) && conf.getBoolVar(ConfVars.HIVETESTMODEROLLBACKTXN)) { diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 70fce3266c..76ca5e2355 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -108,6 +108,7 @@ import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.hooks.WriteEntity; +import org.apache.hadoop.hive.ql.io.AcidInputFormat; import org.apache.hadoop.hive.ql.io.AcidOutputFormat; import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.io.AcidUtils.Operation; @@ -119,6 +120,7 @@ import org.apache.hadoop.hive.ql.lib.Dispatcher; import org.apache.hadoop.hive.ql.lib.GraphWalker; import org.apache.hadoop.hive.ql.lib.Node; +import org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; import org.apache.hadoop.hive.ql.metadata.DummyPartition; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -12202,14 +12204,6 @@ private void validate(Task task, boolean reworkMapredWor validate(childTask, reworkMapredWork); } } - - /** - * Get the row resolver given an operator. - */ - public RowResolver getRowResolver(Operator opt) { - return opParseCtx.get(opt).getRowResolver(); - } - /** * Add default properties for table property. If a default parameter exists * in the tblProp, the value in tblProp will be kept. @@ -12219,7 +12213,8 @@ public RowResolver getRowResolver(Operator opt) { * @return Modified table property map */ private Map addDefaultProperties( - Map tblProp, boolean isExt, StorageFormat storageFormat) { + Map tblProp, boolean isExt, StorageFormat storageFormat, + String qualifiedTableName, List sortCols) { Map retValue; if (tblProp == null) { retValue = new HashMap(); @@ -12238,16 +12233,45 @@ public RowResolver getRowResolver(Operator opt) { } } } - if (HiveConf.getBoolVar(conf, ConfVars.HIVE_CREATE_TABLES_AS_INSERT_ONLY) + boolean makeInsertOnly = HiveConf.getBoolVar(conf, ConfVars.HIVE_CREATE_TABLES_AS_INSERT_ONLY); + boolean makeAcid = HiveConf.getBoolVar(conf, ConfVars.HIVE_CREATE_TABLES_AS_ACID) && + HiveConf.getBoolVar(conf, ConfVars.HIVE_SUPPORT_CONCURRENCY) && + DbTxnManager.class.getCanonicalName().equals(HiveConf.getVar(conf, ConfVars.HIVE_TXN_MANAGER)); + if ((makeInsertOnly || makeAcid) && !isExt && StringUtils.isBlank(storageFormat.getStorageHandler()) + //don't overwrite user choice if transactional attribute is explicitly set && !retValue.containsKey(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL)) { - retValue.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, "true"); - String oldProps = retValue.get(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES); - if (oldProps != null) { - LOG.warn("Non-transactional table has transactional properties; overwriting " + oldProps); + if(makeInsertOnly) { + retValue.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, "true"); + retValue.put(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES, + TransactionalValidationListener.INSERTONLY_TRANSACTIONAL_PROPERTY); + } + if(makeAcid) { + /*for CTAS, TransactionalValidationListener.makeAcid() runs to late to make table Acid + so the initial write ends up running as non-acid...*/ + try { + Class inputFormatClass = storageFormat.getInputFormat() == null ? null : + Class.forName(storageFormat.getInputFormat()); + Class outputFormatClass = storageFormat.getOutputFormat() == null ? null : + Class.forName(storageFormat.getOutputFormat()); + if (inputFormatClass == null || outputFormatClass == null || + !AcidInputFormat.class.isAssignableFrom(inputFormatClass) || + !AcidOutputFormat.class.isAssignableFrom(outputFormatClass)) { + return retValue; + } + } catch (ClassNotFoundException e) { + LOG.warn("Could not verify InputFormat=" + storageFormat.getInputFormat() + " or OutputFormat=" + + storageFormat.getOutputFormat() + " for " + qualifiedTableName); + return retValue; + } + if(sortCols != null && !sortCols.isEmpty()) { + return retValue; + } + retValue.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, "true"); + retValue.put(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES, + TransactionalValidationListener.DEFAULT_TRANSACTIONAL_PROPERTY); + LOG.info("Automatically chose to make " + qualifiedTableName + " acid."); } - retValue.put(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES, - TransactionalValidationListener.INSERTONLY_TRANSACTIONAL_PROPERTY); } return retValue; } @@ -12472,7 +12496,7 @@ ASTNode analyzeCreateTable( switch (command_type) { case CREATE_TABLE: // REGULAR CREATE TABLE DDL - tblProps = addDefaultProperties(tblProps, isExt, storageFormat); + tblProps = addDefaultProperties(tblProps, isExt, storageFormat, dbDotTab, sortCols); CreateTableDesc crtTblDesc = new CreateTableDesc(dbDotTab, isExt, isTemporary, cols, partCols, bucketCols, sortCols, numBuckets, rowFormatParams.fieldDelim, @@ -12493,7 +12517,7 @@ ASTNode analyzeCreateTable( break; case CTLT: // create table like - tblProps = addDefaultProperties(tblProps, isExt, storageFormat); + tblProps = addDefaultProperties(tblProps, isExt, storageFormat, dbDotTab, sortCols); if (isTemporary) { Table likeTable = getTable(likeTableName, false); @@ -12570,7 +12594,7 @@ ASTNode analyzeCreateTable( } } - tblProps = addDefaultProperties(tblProps, isExt, storageFormat); + tblProps = addDefaultProperties(tblProps, isExt, storageFormat, dbDotTab, sortCols); tableDesc = new CreateTableDesc(qualifiedTabName[0], dbDotTab, isExt, isTemporary, cols, partCols, bucketCols, sortCols, numBuckets, rowFormatParams.fieldDelim, rowFormatParams.fieldEscape, rowFormatParams.collItemDelim, rowFormatParams.mapKeyDelim, diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java index df14752613..cdd09ba465 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.metastore; import java.io.IOException; +import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -34,6 +35,7 @@ import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.events.PreAlterTableEvent; import org.apache.hadoop.hive.metastore.events.PreCreateTableEvent; import org.apache.hadoop.hive.metastore.events.PreEventContext; @@ -199,6 +201,58 @@ private void checkSorted(Table newTable) throws MetaException { } /** + * Want to make a a newly create table Acid (unless it explicitly has transactional=false param) + * if table can support it. Also see SemanticAnalyzer.addDefaultProperties() which performs the + * same logic. This code path is more general since it is activated even if you create a table + * via Thrift, WebHCat etc but some operations like CTAS create the table (metastore object) as + * the last step (i.e. after the data is written) but write itself is has to be aware of the type + * of table so this Listener is too late. + */ + private void makeAcid(Table newTable) throws MetaException { + if(true) return; + if(newTable.getParameters() != null && + newTable.getParameters().containsKey(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL)) { + LOG.info("Could not make " + Warehouse.getQualifiedName(newTable) + " acid: already has " + + hive_metastoreConstants.TABLE_IS_TRANSACTIONAL + "=" + + newTable.getParameters().get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL)); + return; + } + Configuration conf = MetastoreConf.newMetastoreConf(); + boolean makeAcid = + //no point making an acid table if these other props are not set since it will just throw + //exceptions when someone tries to use the table. + MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.CREATE_TABLES_AS_ACID) && + MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.HIVE_SUPPORT_CONCURRENCY) && + "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager".equals( + MetastoreConf.getVar(conf, MetastoreConf.ConfVars.HIVE_TXN_MANAGER) + ); + + if(makeAcid) { + if(!conformToAcid(newTable)) { + LOG.info("Could not make " + Warehouse.getQualifiedName(newTable) + " acid: wrong IO format"); + return; + } + if(!TableType.MANAGED_TABLE.toString().equalsIgnoreCase(newTable.getTableType())) { + //todo should this check be in conformToAcid()? + LOG.info("Could not make " + Warehouse.getQualifiedName(newTable) + " acid: it's " + + newTable.getTableType()); + return; + } + if(newTable.getSd().getSortColsSize() > 0) { + LOG.info("Could not make " + Warehouse.getQualifiedName(newTable) + " acid: it's sorted"); + return; + } + //check if orc and not sorted + Map parameters = newTable.getParameters(); + if (parameters == null || parameters.isEmpty()) { + parameters = new HashMap<>(); + } + parameters.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, "true"); + newTable.setParameters(parameters); + LOG.info("Automatically chose to make " + Warehouse.getQualifiedName(newTable) + " acid."); + } + } + /** * Normalize case and make sure: * 1. 'true' is the only value to be set for 'transactional' (if set at all) * 2. If set to 'true', we should also enforce bucketing and ORC format @@ -207,6 +261,7 @@ private void handleCreateTableTransactionalProp(PreCreateTableEvent context) thr Table newTable = context.getTable(); Map parameters = newTable.getParameters(); if (parameters == null || parameters.isEmpty()) { + makeAcid(newTable); return; } String transactional = null; @@ -226,6 +281,7 @@ private void handleCreateTableTransactionalProp(PreCreateTableEvent context) thr } if (transactional == null) { + makeAcid(newTable); return; } @@ -279,7 +335,7 @@ private void normazlieTransactionalPropertyDefault(Table table) { * Check that InputFormatClass/OutputFormatClass should implement * AcidInputFormat/AcidOutputFormat */ - public static boolean conformToAcid(Table table) throws MetaException { + private static boolean conformToAcid(Table table) throws MetaException { StorageDescriptor sd = table.getSd(); try { Class inputFormatClass = sd.getInputFormat() == null ? null : diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index b46cc38a22..2e43dc85ea 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -53,7 +53,7 @@ /** * A set of definitions of config values used by the Metastore. One of the key aims of this * class is to provide backwards compatibility with existing Hive configuration keys while - * allowing the metastore to have its own, Hive independant keys. For this reason access to the + * allowing the metastore to have its own, Hive independent keys. For this reason access to the * underlying Configuration object should always be done via the static methods provided here * rather than directly via {@link Configuration#get(String)} and * {@link Configuration#set(String, String)}. All the methods of this class will handle checking @@ -373,6 +373,9 @@ public static ConfVars getMetaConf(String name) { CONNECTION_USER_NAME("javax.jdo.option.ConnectionUserName", "javax.jdo.option.ConnectionUserName", "APP", "Username to use against metastore database"), + CREATE_TABLES_AS_ACID("metastore.create.as.acid", "hive.create.as.acid", false, + "Whether the eligible tables should be created as full ACID by default. Does \n" + + "not apply to external tables, the ones using storage handlers, etc."), COUNT_OPEN_TXNS_INTERVAL("metastore.count.open.txns.interval", "hive.count.open.txns.interval", 1, TimeUnit.SECONDS, "Time in seconds between checks to count open transactions."), DATANUCLEUS_AUTOSTART("datanucleus.autoStartMechanismMode",