diff --git a/data/conf/spark/yarn-client/hive-site.xml b/data/conf/spark/yarn-client/hive-site.xml index 1617976..ea6fdf4 100644 --- a/data/conf/spark/yarn-client/hive-site.xml +++ b/data/conf/spark/yarn-client/hive-site.xml @@ -241,7 +241,22 @@ spark.executor.memory - 512m + 1g + + + + spark.yarn.executor.memoryOverhead + 0 + + + + spark.driver.memory + 1g + + + + spark.yarn.driver.memoryOverhead + 0 diff --git a/pom.xml b/pom.xml index 22b4ca8..be90d84 100644 --- a/pom.xml +++ b/pom.xml @@ -162,7 +162,7 @@ 4.0.4 0.8.1-alpha 2.2.0 - 1.5.0 + 1.6.0 2.10 2.10.4 1.1 diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java index 0b5c092..0670bc3 100644 --- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java +++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java @@ -526,6 +526,8 @@ public MiniSparkShim(Configuration conf, int numberOfTaskTrackers, mr = new MiniSparkOnYARNCluster("sparkOnYarn"); conf.set("fs.defaultFS", nameNode); conf.set("yarn.resourcemanager.scheduler.class", "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler"); + // disable resource monitoring, although it should be off by default + conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING, false); configureImpersonation(conf); mr.init(conf); mr.start();