From 211b38dc498f5f10e0fb1f25b60273c56d7e1c8e Mon Sep 17 00:00:00 2001 From: Ashutosh Chauhan Date: Thu, 21 Jan 2016 14:07:19 -0800 Subject: [PATCH] HIVE-12897 : Improve dynamic partition loading --- .../java/org/apache/hadoop/hive/conf/HiveConf.java | 17 +++++++------- .../hadoop/hive/metastore/MetaStoreUtils.java | 2 +- .../apache/hadoop/hive/metastore/ObjectStore.java | 7 ++++-- .../hadoop/hive/ql/exec/FileSinkOperator.java | 8 +++++++ .../org/apache/hadoop/hive/ql/metadata/Hive.java | 14 ++++-------- .../index/RewriteParseContextGenerator.java | 14 ++++++++---- .../hadoop/hive/ql/parse/BaseSemanticAnalyzer.java | 4 ++-- .../hadoop/hive/ql/parse/CalcitePlanner.java | 9 ++++---- .../hive/ql/parse/ColumnStatsSemanticAnalyzer.java | 2 +- .../ql/parse/ExplainSQRewriteSemanticAnalyzer.java | 5 ++--- .../hive/ql/parse/ExplainSemanticAnalyzer.java | 5 +++-- .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 26 +++++++++++----------- .../ql/parse/UpdateDeleteSemanticAnalyzer.java | 8 +++---- .../hadoop/hive/ql/plan/DynamicPartitionCtx.java | 14 +++++++++++- .../dynamic_partitions_with_whitelist.q.out | 4 ++-- 15 files changed, 82 insertions(+), 57 deletions(-) diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 97fe7bc..74a8749 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -210,7 +210,8 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { public static final HiveConf.ConfVars[] metaConfVars = { HiveConf.ConfVars.METASTORE_TRY_DIRECT_SQL, HiveConf.ConfVars.METASTORE_TRY_DIRECT_SQL_DDL, - HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT + HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT, + HiveConf.ConfVars.METASTORE_PARTITION_NAME_WHITELIST_PATTERN }; static { @@ -1574,32 +1575,32 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { HIVE_COMPACTOR_ABORTEDTXN_THRESHOLD("hive.compactor.abortedtxn.threshold", 1000, "Number of aborted transactions involving a given table or partition that will trigger\n" + "a major compaction."), - + COMPACTOR_INITIATOR_FAILED_THRESHOLD("hive.compactor.initiator.failed.compacts.threshold", 2, new RangeValidator(1, 20), "Number of consecutive compaction failures (per table/partition) " + "after which automatic compactions will not be scheduled any more. Note that this must be less " + "than hive.compactor.history.retention.failed."), - + HIVE_COMPACTOR_CLEANER_RUN_INTERVAL("hive.compactor.cleaner.run.interval", "5000ms", new TimeValidator(TimeUnit.MILLISECONDS), "Time between runs of the cleaner thread"), COMPACTOR_JOB_QUEUE("hive.compactor.job.queue", "", "Used to specify name of Hadoop queue to which\n" + "Compaction jobs will be submitted. Set to empty string to let Hadoop choose the queue."), - + COMPACTOR_HISTORY_RETENTION_SUCCEEDED("hive.compactor.history.retention.succeeded", 3, new RangeValidator(0, 100), "Determines how many successful compaction records will be " + "retained in compaction history for a given table/partition."), - + COMPACTOR_HISTORY_RETENTION_FAILED("hive.compactor.history.retention.failed", 3, new RangeValidator(0, 100), "Determines how many failed compaction records will be " + "retained in compaction history for a given table/partition."), - + COMPACTOR_HISTORY_RETENTION_ATTEMPTED("hive.compactor.history.retention.attempted", 2, new RangeValidator(0, 100), "Determines how many attempted compaction records will be " + "retained in compaction history for a given table/partition."), - + COMPACTOR_HISTORY_REAPER_INTERVAL("hive.compactor.history.reaper.interval", "2m", new TimeValidator(TimeUnit.MILLISECONDS), "Determines how often compaction history reaper runs"), - + HIVE_TIMEDOUT_TXN_REAPER_START("hive.timedout.txn.reaper.start", "100s", new TimeValidator(TimeUnit.MILLISECONDS), "Time delay of 1st reaper run after metastore start"), HIVE_TIMEDOUT_TXN_REAPER_INTERVAL("hive.timedout.txn.reaper.interval", "180s", diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java index eee7f1b..c8859f3 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java @@ -1618,7 +1618,7 @@ public static boolean compareFieldColumns(List schema1, List partVals, + public static String getPartitionValWithInvalidCharacter(List partVals, Pattern partitionValidationPattern) { if (partitionValidationPattern == null) { return null; diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index e044c73..8cebe93 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -290,7 +290,7 @@ public void setConf(Configuration conf) { String partitionValidationRegex = hiveConf.get(HiveConf.ConfVars.METASTORE_PARTITION_NAME_WHITELIST_PATTERN.name()); - if (partitionValidationRegex != null && partitionValidationRegex.equals("")) { + if (partitionValidationRegex != null && !partitionValidationRegex.equals("")) { partitionValidationPattern = Pattern.compile(partitionValidationRegex); } else { partitionValidationPattern = null; @@ -759,7 +759,7 @@ public boolean dropDatabase(String dbname) throws NoSuchObjectException, MetaExc String queryStr = "select name from org.apache.hadoop.hive.metastore.model.MDatabase"; Query query = null; - + openTransaction(); try { query = pm.newQuery(queryStr); @@ -1054,14 +1054,17 @@ public Table getTable(String dbName, String tableName) throws MetaException { return tbls; } + @Override public int getDatabaseCount() throws MetaException { return getObjectCount("name", MDatabase.class.getName()); } + @Override public int getPartitionCount() throws MetaException { return getObjectCount("partitionName", MPartition.class.getName()); } + @Override public int getTableCount() throws MetaException { return getObjectCount("tableName", MTable.class.getName()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java index 3289cfc..1b8872e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.io.AcidUtils; @@ -696,6 +697,13 @@ public void process(Object row, int tag) throws HiveException { } } + String invalidPartitionVal; + if((invalidPartitionVal = MetaStoreUtils.getPartitionValWithInvalidCharacter(dpVals, dpCtx.getWhiteListPattern()))!=null) { + throw new HiveFatalException("Partition value '" + invalidPartitionVal + + "' contains a character " + "not matched by whitelist pattern '" + + dpCtx.getWhiteListPattern().toString() + "'. " + "(configure with " + + HiveConf.ConfVars.METASTORE_PARTITION_NAME_WHITELIST_PATTERN.varname + ")"); + } fpaths = getDynOutPaths(dpVals, lbDirName); // use SubStructObjectInspector to serialize the non-partitioning columns in the input row diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index efb50b2..50681c1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -1427,7 +1427,7 @@ public void loadPartition(Path loadPath, String tableName, * @param isSrcLocal * If the source directory is LOCAL * @param isAcid true if this is an ACID operation - * @throws JSONException + * @throws JSONException */ public Partition loadPartition(Path loadPath, Table tbl, Map partSpec, boolean replace, @@ -1622,7 +1622,7 @@ private void constructOneLBLocationMap(FileStatus fSta, * @param txnId txnId, can be 0 unless isAcid == true * @return partition map details (PartitionSpec and Partition) * @throws HiveException - * @throws JSONException + * @throws JSONException */ public Map, Partition> loadDynamicPartitions(Path loadPath, String tableName, Map partSpec, boolean replace, @@ -1635,16 +1635,10 @@ private void constructOneLBLocationMap(FileStatus fSta, LinkedHashMap, Partition>(); FileSystem fs = loadPath.getFileSystem(conf); - FileStatus[] leafStatus = HiveStatsUtils.getFileStatusRecurse(loadPath, numDP+1, fs); + FileStatus[] leafStatus = HiveStatsUtils.getFileStatusRecurse(loadPath, numDP, fs); // Check for empty partitions for (FileStatus s : leafStatus) { - try { - validatePartitionNameCharacters( - Warehouse.getPartValuesFromPartName(s.getPath().getParent().toString())); - } catch (MetaException e) { - throw new HiveException(e); - } - validPartitions.add(s.getPath().getParent()); + validPartitions.add(s.getPath()); } int partsToLoad = validPartitions.size(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java index 48105de..5e41dee 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.ParseContext; @@ -84,6 +85,11 @@ "tree for input command - " + command + " " , e); LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e)); throw new SemanticException(e.getMessage(), e); + } catch (HiveException e) { + LOG.error("HiveException in generating the operator " + + "tree for input command - " + command + " " , e); + LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e)); + throw new SemanticException(e.getMessage(), e); } return operatorTree; } @@ -96,15 +102,15 @@ * @param sem * @param ast * @return - * @throws SemanticException + * @throws HiveException */ private static Operator doSemanticAnalysis(SemanticAnalyzer sem, - ASTNode ast, Context ctx) throws SemanticException { + ASTNode ast, Context ctx) throws HiveException { QB qb = new QB(null, null, false); ASTNode child = ast; - ParseContext subPCtx = ((SemanticAnalyzer) sem).getParseContext(); + ParseContext subPCtx = sem.getParseContext(); subPCtx.setContext(ctx); - ((SemanticAnalyzer) sem).initParseCtx(subPCtx); + sem.initParseCtx(subPCtx); LOG.info("Starting Sub-query Semantic Analysis"); sem.doPhase1(child, qb, sem.initPhase1Ctx(), null); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index af1ee20..3963457 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -224,7 +224,7 @@ protected static Hive createHiveDB(HiveConf conf) throws SemanticException { return idToTableNameMap; } - public abstract void analyzeInternal(ASTNode ast) throws SemanticException; + public abstract void analyzeInternal(ASTNode ast) throws SemanticException, HiveException; public void init(boolean clearPartsCache) { //no-op } @@ -233,7 +233,7 @@ public void initCtx(Context ctx) { this.ctx = ctx; } - public void analyze(ASTNode ast, Context ctx) throws SemanticException { + public void analyze(ASTNode ast, Context ctx) throws HiveException { initCtx(ctx); init(true); analyzeInternal(ast); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 3fefbd7..a9178ea 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -110,6 +110,7 @@ import org.apache.hadoop.hive.ql.exec.RowSchema; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException; @@ -223,7 +224,7 @@ public CalcitePlanner(HiveConf conf) throws SemanticException { @Override @SuppressWarnings("nls") - public void analyzeInternal(ASTNode ast) throws SemanticException { + public void analyzeInternal(ASTNode ast) throws HiveException { if (runCBO) { PreCboCtx cboCtx = new PreCboCtx(); super.analyzeInternal(ast, cboCtx); @@ -234,7 +235,7 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { @Override @SuppressWarnings("rawtypes") - Operator genOPTree(ASTNode ast, PlannerContext plannerCtx) throws SemanticException { + Operator genOPTree(ASTNode ast, PlannerContext plannerCtx) throws HiveException { Operator sinkOp = null; boolean skipCalcitePlan = false; @@ -667,9 +668,9 @@ ASTNode getOptimizedAST() throws SemanticException { * Get Optimized Hive Operator DAG for the given QB tree in the semAnalyzer. * * @return Optimized Hive operator tree - * @throws SemanticException + * @throws HiveException */ - Operator getOptimizedHiveOPDag() throws SemanticException { + Operator getOptimizedHiveOPDag() throws HiveException { RelNode optimizedOptiqPlan = null; CalcitePlannerAction calcitePlannerAction = new CalcitePlannerAction(prunedPartitions); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java index 1f30cbd..b3eaef4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java @@ -359,7 +359,7 @@ private void checkForPartitionColumns(List specifiedCols, List p } @Override - public void analyze(ASTNode ast, Context origCtx) throws SemanticException { + public void analyze(ASTNode ast, Context origCtx) throws HiveException { QB qb; QBParseInfo qbp; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSQRewriteSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSQRewriteSemanticAnalyzer.java index 2c2339a..8453236 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSQRewriteSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSQRewriteSemanticAnalyzer.java @@ -17,14 +17,13 @@ */ package org.apache.hadoop.hive.ql.parse; -import java.io.Serializable; import java.util.List; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.exec.ExplainSQRewriteTask; -import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; +import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.ExplainSQRewriteWork; public class ExplainSQRewriteSemanticAnalyzer extends BaseSemanticAnalyzer { @@ -36,7 +35,7 @@ public ExplainSQRewriteSemanticAnalyzer(HiveConf conf) throws SemanticException @SuppressWarnings("unchecked") @Override - public void analyzeInternal(ASTNode ast) throws SemanticException { + public void analyzeInternal(ASTNode ast) throws HiveException { ctx.setExplain(true); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java index c1e9ec1..a921beb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hive.ql.exec.ExplainTask; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; +import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.ExplainWork; /** @@ -43,7 +44,7 @@ public ExplainSemanticAnalyzer(HiveConf conf) throws SemanticException { @SuppressWarnings("unchecked") @Override - public void analyzeInternal(ASTNode ast) throws SemanticException { + public void analyzeInternal(ASTNode ast) throws HiveException { boolean extended = false; boolean formatted = false; @@ -79,7 +80,7 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { if (tasks == null) { tasks = Collections.emptyList(); } - + FetchTask fetchTask = sem.getFetchTask(); if (fetchTask != null) { // Initialize fetch work such that operator tree will be constructed. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 5ff90a6..8cc41e1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -2634,7 +2634,7 @@ private void extractJoinCondsFromWhereClause(QBJoinTree joinTree, QB qb, String @SuppressWarnings("nls") private Operator genHavingPlan(String dest, QB qb, Operator input, Map aliasToOpInfo) - throws SemanticException { + throws HiveException { ASTNode havingExpr = qb.getParseInfo().getHavingForClause(dest); @@ -2660,7 +2660,7 @@ private Operator genHavingPlan(String dest, QB qb, Operator input, private Operator genPlanForSubQueryPredicate( QB qbSQ, - ISubQueryJoinInfo subQueryPredicate) throws SemanticException { + ISubQueryJoinInfo subQueryPredicate) throws HiveException { qbSQ.setSubQueryDef(subQueryPredicate.getSubQuery()); Phase1Ctx ctx_1 = initPhase1Ctx(); doPhase1(subQueryPredicate.getSubQueryAST(), qbSQ, ctx_1, null); @@ -2673,7 +2673,7 @@ private Operator genPlanForSubQueryPredicate( private Operator genFilterPlan(ASTNode searchCond, QB qb, Operator input, Map aliasToOpInfo, boolean forHavingClause, boolean forGroupByClause) - throws SemanticException { + throws HiveException { OpParseContext inputCtx = opParseCtx.get(input); RowResolver inputRR = inputCtx.getRowResolver(); @@ -5365,7 +5365,7 @@ private Operator genGroupByPlan1MR(String dest, QB qb, Operator input) @SuppressWarnings({"nls"}) private Operator genGroupByPlan1ReduceMultiGBY(List dests, QB qb, Operator input, Map aliasToOpInfo) - throws SemanticException { + throws HiveException { QBParseInfo parseInfo = qb.getParseInfo(); @@ -6179,7 +6179,7 @@ private void genPartnCols(String dest, Operator input, QB qb, @SuppressWarnings("nls") protected Operator genFileSinkPlan(String dest, QB qb, Operator input) - throws SemanticException { + throws HiveException { RowResolver inputRR = opParseCtx.get(input).getRowResolver(); QBMetaData qbm = qb.getMetaData(); @@ -8744,7 +8744,7 @@ private boolean matchExprLists(List list1, List list @SuppressWarnings("nls") private Operator genBodyPlan(QB qb, Operator input, Map aliasToOpInfo) - throws SemanticException { + throws HiveException { QBParseInfo qbp = qb.getParseInfo(); TreeSet ks = new TreeSet(qbp.getClauseNames()); @@ -8876,7 +8876,7 @@ private Operator genBodyPlan(QB qb, Operator input, Map aliasT private Operator genPostGroupByBodyPlan(Operator curr, String dest, QB qb, Map aliasToOpInfo, Operator gbySource) - throws SemanticException { + throws HiveException { QBParseInfo qbp = qb.getParseInfo(); @@ -9574,7 +9574,7 @@ private void setupStats(TableScanDesc tsDesc, QBParseInfo qbp, Table tab, String } } - private Operator genPlan(QB parent, QBExpr qbexpr) throws SemanticException { + private Operator genPlan(QB parent, QBExpr qbexpr) throws HiveException { if (qbexpr.getOpcode() == QBExpr.Opcode.NULLOP) { boolean skipAmbiguityCheck = viewSelect == null && parent.isTopLevelSelectStarQuery(); return genPlan(qbexpr.getQB(), skipAmbiguityCheck); @@ -9589,13 +9589,13 @@ private Operator genPlan(QB parent, QBExpr qbexpr) throws SemanticException { return null; } - public Operator genPlan(QB qb) throws SemanticException { + public Operator genPlan(QB qb) throws HiveException { return genPlan(qb, false); } @SuppressWarnings("nls") public Operator genPlan(QB qb, boolean skipAmbiguityCheck) - throws SemanticException { + throws HiveException { // First generate all the opInfos for the elements in the from clause // Must be deterministic order map - see HIVE-8707 @@ -9953,7 +9953,7 @@ boolean analyzeCreateTable(ASTNode child) throws SemanticException { @Override @SuppressWarnings("nls") - public void analyzeInternal(ASTNode ast) throws SemanticException { + public void analyzeInternal(ASTNode ast) throws HiveException { analyzeInternal(ast, new PlannerContext()); } @@ -10085,11 +10085,11 @@ private void preProcessForInsert(ASTNode node, QB qb) throws SemanticException { throw new SemanticException(ex); } } - Operator genOPTree(ASTNode ast, PlannerContext plannerCtx) throws SemanticException { + Operator genOPTree(ASTNode ast, PlannerContext plannerCtx) throws HiveException { return genPlan(qb); } - void analyzeInternal(ASTNode ast, PlannerContext plannerCtx) throws SemanticException { + void analyzeInternal(ASTNode ast, PlannerContext plannerCtx) throws HiveException { // 1. Generate Resolved Parse tree from syntax tree LOG.info("Starting Semantic Analysis"); if (!genResolvedParseTree(ast, plannerCtx)) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java index 5b4365c..123efdb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java @@ -57,7 +57,7 @@ public UpdateDeleteSemanticAnalyzer(HiveConf conf) throws SemanticException { } @Override - public void analyzeInternal(ASTNode tree) throws SemanticException { + public void analyzeInternal(ASTNode tree) throws HiveException { if (useSuper) { super.analyzeInternal(tree); } else { @@ -91,17 +91,17 @@ protected boolean deleting() { return ctx.getAcidOperation() == AcidUtils.Operation.DELETE; } - private void analyzeUpdate(ASTNode tree) throws SemanticException { + private void analyzeUpdate(ASTNode tree) throws HiveException { ctx.setAcidOperation(AcidUtils.Operation.UPDATE); reparseAndSuperAnalyze(tree); } - private void analyzeDelete(ASTNode tree) throws SemanticException { + private void analyzeDelete(ASTNode tree) throws HiveException { ctx.setAcidOperation(AcidUtils.Operation.DELETE); reparseAndSuperAnalyze(tree); } - private void reparseAndSuperAnalyze(ASTNode tree) throws SemanticException { + private void reparseAndSuperAnalyze(ASTNode tree) throws HiveException { List children = tree.getChildren(); // The first child should be the table we are deleting from ASTNode tabName = (ASTNode)children.get(0); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPartitionCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPartitionCtx.java index 95d5635..ba3c266 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPartitionCtx.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPartitionCtx.java @@ -21,9 +21,13 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.regex.Pattern; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Table; public class DynamicPartitionCtx implements Serializable { @@ -44,12 +48,13 @@ private List dpNames; // dp column names private String defaultPartName; // default partition name in case of null or empty value private int maxPartsPerNode; // maximum dynamic partitions created per mapper/reducer + private Pattern whiteListPattern; public DynamicPartitionCtx() { } public DynamicPartitionCtx(Table tbl, Map partSpec, String defaultPartName, - int maxParts) { + int maxParts) throws HiveException { this.partSpec = partSpec; this.spNames = new ArrayList(); this.dpNames = new ArrayList(); @@ -71,6 +76,8 @@ public DynamicPartitionCtx(Table tbl, Map partSpec, String defau } else { this.spPath = null; } + String confVal = Hive.get().getMetaConf(ConfVars.METASTORE_PARTITION_NAME_WHITELIST_PATTERN.varname); + this.whiteListPattern = (confVal != null && !"".equals(confVal)) ? Pattern.compile(confVal) : null; } public DynamicPartitionCtx(DynamicPartitionCtx dp) { @@ -84,6 +91,11 @@ public DynamicPartitionCtx(DynamicPartitionCtx dp) { this.dpNames = dp.dpNames; this.defaultPartName = dp.defaultPartName; this.maxPartsPerNode = dp.maxPartsPerNode; + this.whiteListPattern = dp.whiteListPattern; + } + + public Pattern getWhiteListPattern() { + return whiteListPattern; } public int getMaxPartitionsPerNode() { diff --git a/ql/src/test/results/clientnegative/dynamic_partitions_with_whitelist.q.out b/ql/src/test/results/clientnegative/dynamic_partitions_with_whitelist.q.out index f069ae8..654d892 100644 --- a/ql/src/test/results/clientnegative/dynamic_partitions_with_whitelist.q.out +++ b/ql/src/test/results/clientnegative/dynamic_partitions_with_whitelist.q.out @@ -32,5 +32,5 @@ PREHOOK: type: QUERY PREHOOK: Input: default@source_table PREHOOK: Input: default@source_table@ds=2008-04-08/hr=11 PREHOOK: Output: default@dest_table -Failed with exception MetaException(message:Partition value 'val_129' contains a character not matched by whitelist pattern '[^9]*'. (configure with hive.metastore.partition.name.whitelist.pattern)) -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.MoveTask +#### A masked pattern was here #### +FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask -- 1.7.12.4 (Apple Git-37)