diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java index 3d682a0..b668246 100644 --- a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java +++ b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java @@ -75,6 +75,7 @@ private static final String EXECUTOR_OPTS_KEY = "spark.executor.extraJavaOptions"; private static final String DRIVER_EXTRA_CLASSPATH = "spark.driver.extraClassPath"; private static final String EXECUTOR_EXTRA_CLASSPATH = "spark.executor.extraClassPath"; + private static final String SPARK_YARN_REPORT_INTERVAL = "spark.yarn.report.interval"; private final Map conf; private final HiveConf hiveConf; @@ -302,6 +303,15 @@ public void run() { } } + + //The application reports tend to spam the hive logs. This is controlled by spark, and the default seems to be 1s. + //If it is not specified, set it to a much higher number. It can always be overriden by user. + String sparkYarnReportInterval = conf.get(SPARK_YARN_REPORT_INTERVAL); + if (sparkYarnReportInterval == null) { + //the new version of spark also takes time-units, but old versions do not. + allProps.put(SPARK_YARN_REPORT_INTERVAL, "60000"); + } + Writer writer = new OutputStreamWriter(new FileOutputStream(properties), Charsets.UTF_8); try { allProps.store(writer, "Spark Context configuration");