From eb5ce0026c69ff1cc5eb837df547d95d9f54dea9 Mon Sep 17 00:00:00 2001 From: Ashutosh Chauhan Date: Tue, 31 May 2016 23:15:38 -0700 Subject: [PATCH] HIVE-13902 : [Refactor] Minimize metastore jar dependencies on task nodes --- .../org/apache/hive/common/util/HiveStringUtils.java | 17 +++++++++++++++++ .../apache/hadoop/hive/metastore/MetaStoreUtils.java | 20 +++----------------- .../apache/hadoop/hive/ql/exec/FileSinkOperator.java | 4 ++-- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/common/src/java/org/apache/hive/common/util/HiveStringUtils.java b/common/src/java/org/apache/hive/common/util/HiveStringUtils.java index 6d28396..bba14e2 100644 --- a/common/src/java/org/apache/hive/common/util/HiveStringUtils.java +++ b/common/src/java/org/apache/hive/common/util/HiveStringUtils.java @@ -38,9 +38,11 @@ import java.util.Locale; import java.util.Properties; import java.util.StringTokenizer; +import java.util.regex.Pattern; import com.google.common.collect.Interner; import com.google.common.collect.Interners; + import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.classification.InterfaceAudience; import org.apache.hadoop.hive.common.classification.InterfaceStability; @@ -958,4 +960,19 @@ public static boolean commentProvided(char[] chars) { } return false; } + + public static String getPartitionValWithInvalidCharacter(List partVals, + Pattern partitionValidationPattern) { + if (partitionValidationPattern == null) { + return null; + } + + for (String partVal : partVals) { + if (!partitionValidationPattern.matcher(partVal).matches()) { + return partVal; + } + } + + return null; + } } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java index 84b24ab..2f1e460 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java @@ -86,6 +86,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge; +import org.apache.hive.common.util.HiveStringUtils; import org.apache.hive.common.util.ReflectionUtil; import javax.annotation.Nullable; @@ -1631,7 +1632,7 @@ public static void validatePartitionNameCharacters(List partVals, Pattern partitionValidationPattern) throws MetaException { String invalidPartitionVal = - getPartitionValWithInvalidCharacter(partVals, partitionValidationPattern); + HiveStringUtils.getPartitionValWithInvalidCharacter(partVals, partitionValidationPattern); if (invalidPartitionVal != null) { throw new MetaException("Partition value '" + invalidPartitionVal + "' contains a character " + "not matched by whitelist pattern '" + @@ -1642,7 +1643,7 @@ public static void validatePartitionNameCharacters(List partVals, public static boolean partitionNameHasValidCharacters(List partVals, Pattern partitionValidationPattern) { - return getPartitionValWithInvalidCharacter(partVals, partitionValidationPattern) == null; + return HiveStringUtils.getPartitionValWithInvalidCharacter(partVals, partitionValidationPattern) == null; } /** @@ -1692,21 +1693,6 @@ public static boolean compareFieldColumns(List schema1, List partVals, - Pattern partitionValidationPattern) { - if (partitionValidationPattern == null) { - return null; - } - - for (String partVal : partVals) { - if (!partitionValidationPattern.matcher(partVal).matches()) { - return partVal; - } - } - - return null; - } - public static String ARCHIVING_LEVEL = "archiving_level"; public static int getArchivingLevel(Partition part) throws MetaException { if (!isArchived(part)) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java index cd5e7e7..556d273 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java @@ -39,7 +39,6 @@ import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.io.AcidUtils; @@ -79,6 +78,7 @@ import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hive.common.util.HiveStringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -702,7 +702,7 @@ public void process(Object row, int tag) throws HiveException { } String invalidPartitionVal; - if((invalidPartitionVal = MetaStoreUtils.getPartitionValWithInvalidCharacter(dpVals, dpCtx.getWhiteListPattern()))!=null) { + if((invalidPartitionVal = HiveStringUtils.getPartitionValWithInvalidCharacter(dpVals, dpCtx.getWhiteListPattern()))!=null) { throw new HiveFatalException("Partition value '" + invalidPartitionVal + "' contains a character not matched by whitelist pattern '" + dpCtx.getWhiteListPattern().toString() + "'. " + "(configure with " + -- 1.7.12.4 (Apple Git-37)