From 835bdce6f5d6b1bec35f79c541a4d5d8ca3e7201 Mon Sep 17 00:00:00 2001
From: Yiming Liu <liuyiming.vip@gmail.com>
Date: Fri, 24 Jun 2016 15:13:14 +0800
Subject: [PATCH] KYLIN-1818: change kafka dep to provided

---
 assembly/pom.xml                   |  1 -
 build/bin/find-kafka-dependency.sh | 51 ++++++++++++++++++++++++++++++++++++++
 build/bin/kylin.sh                 |  4 +++
 3 files changed, 55 insertions(+), 1 deletion(-)
 create mode 100644 build/bin/find-kafka-dependency.sh

diff --git a/assembly/pom.xml b/assembly/pom.xml
index d643b62..d7764a0 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -177,7 +177,6 @@
             <artifactId>kafka_2.10</artifactId>
             <version>${kafka.version}</version>
             <scope>provided</scope>
-            <!-- FIXME: Should be provided just like hive and hbase, inflates job jar from 9 MB to 21 MB -->
         </dependency>
 
     </dependencies>
diff --git a/build/bin/find-kafka-dependency.sh b/build/bin/find-kafka-dependency.sh
new file mode 100644
index 0000000..c6b9c24
--- /dev/null
+++ b/build/bin/find-kafka-dependency.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+kafka_home=
+
+if [ -n "$KAFKA_HOME" ]
+then
+    echo "KAFKA_HOME is set to: $KAFKA_HOME, use it to locate kafka dependencies."
+    kafka_home=$KAFKA_HOME
+fi
+
+if [ -z "$KAFKA_HOME" ]
+then
+    echo "Couldn't find kafka home. Please set KAFKA_HOME to the path which contains kafka dependencies."
+    exit 1
+fi
+
+# works for kafka 9+
+kafka_client=`find -L "$(dirname $kafka_home)" -name 'kafka-clients-[a-z0-9A-Z\.-]*.jar' ! -name '*doc*' ! -name '*test*' ! -name '*sources*' ''-printf '%p:' | sed 's/:$//'`
+if [ -z "$kafka_client" ]
+then
+# works for kafka 8
+    kafka_broker=`find -L "$(dirname $kafka_home)" -name 'kafka_[a-z0-9A-Z\.-]*.jar' ! -name '*doc*' ! -name '*test*' ! -name '*sources*' ''-printf '%p:' | sed 's/:$//'`
+    if [ -z "$kafka_broker" ]
+    then
+        echo "kafka client lib not found"
+        exit 1
+    else
+        echo "kafka dependency: $kafka_broker"
+        export kafka_dependency
+    fi
+else
+    echo "kafka dependency: $kafka_client"
+    export kafka_dependency
+fi
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index 2ea3a8c..f3710c3 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -141,6 +141,7 @@ then
         #retrive $hive_dependency and $hbase_dependency
         source ${dir}/find-hive-dependency.sh
         source ${dir}/find-hbase-dependency.sh
+        source ${dir}/find-kafka-dependency.sh
         #retrive $KYLIN_EXTRA_START_OPTS
         if [ -f "${dir}/setenv.sh" ]
             then source ${dir}/setenv.sh
@@ -153,6 +154,7 @@ then
         hbase ${KYLIN_EXTRA_START_OPTS} \
         -Dlog4j.configuration=kylin-log4j.properties\
         -Dkylin.hive.dependency=${hive_dependency} \
+        -Dkylin.kafka.dependency=${kafka_dependency} \
         -Dkylin.hbase.dependency=${hbase_dependency} \
         org.apache.kylin.engine.streaming.cli.StreamingCLI $@ > ${KYLIN_HOME}/logs/streaming_$3_$4.log 2>&1 & echo $! > ${KYLIN_HOME}/logs/$3_$4 &
         echo "streaming started name: $3 id: $4"
@@ -187,6 +189,7 @@ then
     #retrive $hive_dependency and $hbase_dependency
     source ${dir}/find-hive-dependency.sh
     source ${dir}/find-hbase-dependency.sh
+    source ${dir}/find-kafka-dependency.sh
     #retrive $KYLIN_EXTRA_START_OPTS
     if [ -f "${dir}/setenv.sh" ]
         then source ${dir}/setenv.sh
@@ -199,6 +202,7 @@ then
     hbase ${KYLIN_EXTRA_START_OPTS} \
     -Dlog4j.configuration=kylin-log4j.properties\
     -Dkylin.hive.dependency=${hive_dependency} \
+    -Dkyiln.kafka.dependency=${kafka_dependency} \
     -Dkylin.hbase.dependency=${hbase_dependency} \
     org.apache.kylin.engine.streaming.cli.MonitorCLI $@ > ${KYLIN_HOME}/logs/monitor.log 2>&1
     exit 0
-- 
2.7.4 (Apple Git-66)