diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 7ea2de9019c..cc8ab4a5302 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -4787,6 +4787,7 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "hive.spark.client.rpc.max.size," + "hive.spark.client.rpc.threads," + "hive.spark.client.secret.bits," + + "hive.query.max.length," + "hive.spark.client.rpc.server.address," + "hive.spark.client.rpc.server.port," + "hive.spark.client.rpc.sasl.mechanisms," + @@ -4823,6 +4824,8 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal SPARK_CLIENT_TYPE.varname, "Comma separated list of variables which are related to remote spark context.\n" + "Changing these variables will result in re-creating the spark session."), + HIVE_QUERY_MAX_LENGTH("hive.query.max.length", "10Mb", new SizeValidator(), "The maximum" + + " size of a query string. Enforced after variable substitutions."), HIVE_QUERY_TIMEOUT_SECONDS("hive.query.timeout.seconds", "0s", new TimeValidator(TimeUnit.SECONDS), "Timeout for Running Query in seconds. A nonpositive value means infinite. " + diff --git common/src/java/org/apache/hadoop/hive/conf/SystemVariables.java common/src/java/org/apache/hadoop/hive/conf/SystemVariables.java index 695f3ec01ac..89ea20eb396 100644 --- common/src/java/org/apache/hadoop/hive/conf/SystemVariables.java +++ common/src/java/org/apache/hadoop/hive/conf/SystemVariables.java @@ -88,6 +88,10 @@ static String substitute(Configuration conf, String expr) { } protected final String substitute(Configuration conf, String expr, int depth) { + long maxLength = 0; + if (conf != null) { + maxLength = HiveConf.getSizeVar(conf, HiveConf.ConfVars.HIVE_QUERY_MAX_LENGTH); + } Matcher match = varPat.matcher(""); String eval = expr; StringBuilder builder = new StringBuilder(); @@ -107,12 +111,18 @@ protected final String substitute(Configuration conf, String expr, int depth) { found = true; } builder.append(eval.substring(prev, match.start())).append(substitute); + if (maxLength > 0 && builder.length() > maxLength) { + throw new IllegalStateException("Query length longer than hive.query.max.length ("+builder.length()+">"+maxLength+")."); + } prev = match.end(); } if (!found) { return eval; } builder.append(eval.substring(prev)); + if (maxLength > 0 && builder.length() > maxLength) { + throw new IllegalStateException("Query length longer than hive.query.max.length ("+builder.length()+">"+maxLength+")."); + } eval = builder.toString(); } if (s > depth) { diff --git common/src/test/org/apache/hadoop/hive/conf/SystemVariablesTest.java common/src/test/org/apache/hadoop/hive/conf/SystemVariablesTest.java new file mode 100644 index 00000000000..299bcc3179a --- /dev/null +++ common/src/test/org/apache/hadoop/hive/conf/SystemVariablesTest.java @@ -0,0 +1,43 @@ +package org.apache.hadoop.hive.conf; + +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.junit.Assert.*; + +public class SystemVariablesTest { + + @Test + public void testSubstituteNotFound() { + + } + + @Test + public void testSubstituteLongSelfReference() { + String randomPart = RandomStringUtils.random(100_000); + String reference = "${hiveconf:myTestVariable}"; + + StringBuilder longStringWithReferences = new StringBuilder(); + for(int i = 0; i < 10; i ++) { + longStringWithReferences.append(randomPart).append(reference); + } + + SystemVariables uut = new SystemVariables(); + + HiveConf conf = new HiveConf(); + conf.set(HiveConf.ConfVars.HIVE_QUERY_MAX_LENGTH.varname, "100Kb"); + conf.set("myTestVariable", longStringWithReferences.toString()); + + try { + uut.substitute(conf, longStringWithReferences.toString(), 40); + } catch (Exception e) { + if (!e.getMessage().startsWith("Query length longer than hive.query.max.length")) { + fail("unexpected error message: " + e.getMessage()); + } + return; + } + fail("should have thrown exception during substitution"); + } +} \ No newline at end of file