diff --git metastore/if/hive_metastore.thrift metastore/if/hive_metastore.thrift index 4625801..2113263 100755 --- metastore/if/hive_metastore.thrift +++ metastore/if/hive_metastore.thrift @@ -1139,6 +1139,7 @@ const string META_TABLE_PARTITION_COLUMN_TYPES = "partition_columns.types", const string FILE_INPUT_FORMAT = "file.inputformat", const string FILE_OUTPUT_FORMAT = "file.outputformat", const string META_TABLE_STORAGE = "storage_handler", - +const string TABLE_IS_TRANSACTIONAL = "transactional", +const string TABLE_NO_AUTO_COMPACT = "no_auto_compaction", diff --git metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp index c1f9da7..c7bf9ba 100644 --- metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp +++ metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp @@ -53,6 +53,10 @@ hive_metastoreConstants::hive_metastoreConstants() { META_TABLE_STORAGE = "storage_handler"; + TABLE_IS_TRANSACTIONAL = "transactional"; + + TABLE_NO_AUTO_COMPACT = "no_auto_compaction"; + } }}} // namespace diff --git metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h index d41e2e8..35a8a50 100644 --- metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h +++ metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h @@ -36,6 +36,8 @@ class hive_metastoreConstants { std::string FILE_INPUT_FORMAT; std::string FILE_OUTPUT_FORMAT; std::string META_TABLE_STORAGE; + std::string TABLE_IS_TRANSACTIONAL; + std::string TABLE_NO_AUTO_COMPACT; }; extern const hive_metastoreConstants g_hive_metastore_constants; diff --git metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java index 8d17378..0f2fca8 100644 --- metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java +++ metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java @@ -75,4 +75,8 @@ public static final String META_TABLE_STORAGE = "storage_handler"; + public static final String TABLE_IS_TRANSACTIONAL = "transactional"; + + public static final String TABLE_NO_AUTO_COMPACT = "no_auto_compaction"; + } diff --git metastore/src/gen/thrift/gen-php/metastore/Types.php metastore/src/gen/thrift/gen-php/metastore/Types.php index 8c3643f..cc2cdad 100644 --- metastore/src/gen/thrift/gen-php/metastore/Types.php +++ metastore/src/gen/thrift/gen-php/metastore/Types.php @@ -12890,4 +12890,8 @@ $GLOBALS['hive_metastore_CONSTANTS']['FILE_OUTPUT_FORMAT'] = "file.outputformat" $GLOBALS['hive_metastore_CONSTANTS']['META_TABLE_STORAGE'] = "storage_handler"; +$GLOBALS['hive_metastore_CONSTANTS']['TABLE_IS_TRANSACTIONAL'] = "transactional"; + +$GLOBALS['hive_metastore_CONSTANTS']['TABLE_NO_AUTO_COMPACT'] = "no_auto_compaction"; + diff --git metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore-remote metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore-remote old mode 100644 new mode 100755 diff --git metastore/src/gen/thrift/gen-py/hive_metastore/constants.py metastore/src/gen/thrift/gen-py/hive_metastore/constants.py index e3e1fde..81f70eb 100644 --- metastore/src/gen/thrift/gen-py/hive_metastore/constants.py +++ metastore/src/gen/thrift/gen-py/hive_metastore/constants.py @@ -30,3 +30,5 @@ FILE_INPUT_FORMAT = "file.inputformat" FILE_OUTPUT_FORMAT = "file.outputformat" META_TABLE_STORAGE = "storage_handler" +TABLE_IS_TRANSACTIONAL = "transactional" +TABLE_NO_AUTO_COMPACT = "no_auto_compaction" diff --git metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb index 162b716..3208ecd 100644 --- metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb +++ metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb @@ -49,3 +49,7 @@ FILE_OUTPUT_FORMAT = %q"file.outputformat" META_TABLE_STORAGE = %q"storage_handler" +TABLE_IS_TRANSACTIONAL = %q"transactional" + +TABLE_NO_AUTO_COMPACT = %q"no_auto_compaction" + diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 88fd0fc..a44dd76 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -51,6 +51,7 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Order; +import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.QueryProperties; import org.apache.hadoop.hive.ql.exec.AbstractMapJoinOperator; @@ -318,9 +319,6 @@ private static final String VALUES_TMP_TABLE_NAME_PREFIX = "Values__Tmp__Table__"; - @VisibleForTesting - static final String ACID_TABLE_PROPERTY = "transactional"; - private HashMap opToPartPruner; private HashMap opToPartList; private HashMap> topOps; @@ -12415,7 +12413,9 @@ else return (ltd.getReplace() ? WriteEntity.WriteType.INSERT_OVERWRITE : private boolean isAcidTable(Table tab) { if (tab == null) return false; if (!SessionState.get().getTxnMgr().supportsAcid()) return false; - return tab.getProperty(ACID_TABLE_PROPERTY) != null; + String tableIsTransactional = + tab.getProperty(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL); + return tableIsTransactional != null && tableIsTransactional.equalsIgnoreCase("true"); } private boolean isAcidOutputFormat(Class of) { diff --git ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java index 5545bf7..a8aab56 100644 --- ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java +++ ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.metastore.txn.CompactionInfo; import org.apache.hadoop.hive.metastore.txn.TxnHandler; import org.apache.hadoop.hive.ql.io.AcidUtils; @@ -52,8 +53,6 @@ static final private String CLASS_NAME = Initiator.class.getName(); static final private Log LOG = LogFactory.getLog(CLASS_NAME); - static final private String NO_COMPACTION = "NO_AUTO_COMPACTION"; - private long checkInterval; @Override @@ -85,9 +84,8 @@ public void run() { try { Table t = resolveTable(ci); // check if no compaction set for this table - if (t.getParameters().get(NO_COMPACTION) != null) { - LOG.info("Table " + tableName(t) + " marked " + NO_COMPACTION + - " so we will not compact it."); + if (noAutoCompactSet(t)) { + LOG.info("Table " + tableName(t) + " marked true so we will not compact it."); continue; } @@ -278,4 +276,16 @@ private void requestCompaction(CompactionInfo ci, String runAs, CompactionType t rqst.setRunas(runAs); txnHandler.compact(rqst); } + + // Because TABLE_NO_AUTO_COMPACT was originally assumed to be NO_AUTO_COMPACT and then was moved + // to no_auto_compact, we need to check it in both cases. + private boolean noAutoCompactSet(Table t) { + String noAutoCompact = + t.getParameters().get(hive_metastoreConstants.TABLE_NO_AUTO_COMPACT); + if (noAutoCompact == null) { + noAutoCompact = + t.getParameters().get(hive_metastoreConstants.TABLE_NO_AUTO_COMPACT.toUpperCase()); + } + return noAutoCompact != null && noAutoCompact.equalsIgnoreCase("true"); + } } diff --git ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java index 87ef193..5f0659b 100644 --- ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java +++ ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java @@ -32,6 +32,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.QueryPlan; import org.apache.hadoop.hive.ql.exec.ExplainTask; @@ -267,7 +268,7 @@ private ReturnInfo parseAndAnalyze(String query, String testName) // I have to create the tables here (rather than in setup()) because I need the Hive // connection, which is conviently created by the semantic analyzer. Map params = new HashMap(1); - params.put(SemanticAnalyzer.ACID_TABLE_PROPERTY, "true"); + params.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, "true"); db.createTable("T", Arrays.asList("a", "b"), null, OrcInputFormat.class, OrcOutputFormat.class, 2, Arrays.asList("a"), params); db.createTable("U", Arrays.asList("a", "b"), Arrays.asList("ds"), OrcInputFormat.class, diff --git ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java index 7d88c1a..b27316d 100644 --- ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java +++ ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java @@ -243,6 +243,32 @@ public void noCompactWhenNoCompactSet() throws Exception { } @Test + public void noCompactWhenNoCompactSetLowerCase() throws Exception { + Map parameters = new HashMap(1); + parameters.put("no_auto_compaction", "true"); + Table t = newTable("default", "ncwncs", false, parameters); + + HiveConf.setIntVar(conf, HiveConf.ConfVars.HIVE_COMPACTOR_ABORTEDTXN_THRESHOLD, 10); + + for (int i = 0; i < 11; i++) { + long txnid = openTxn(); + LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, "default"); + comp.setTablename("ncwncs"); + List components = new ArrayList(1); + components.add(comp); + LockRequest req = new LockRequest(components, "me", "localhost"); + req.setTxnid(txnid); + LockResponse res = txnHandler.lock(req); + txnHandler.abortTxn(new AbortTxnRequest(txnid)); + } + + startInitiator(); + + ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); + Assert.assertEquals(0, rsp.getCompactsSize()); + } + + @Test public void noCompactWhenCompactAlreadyScheduled() throws Exception { Table t = newTable("default", "ncwcas", false);