18/05/09 08:31:21 INFO SparkProcessBuilder: Running '/apache/spark/bin/spark-submit' '--deploy-mode' 'cluster' '--name' 'griffin' '--class' 'org.apache.griffin.measure.Application' '--conf' 'spark.executor.instances=3' '--conf' 'spark.executor.memory=1g' '--conf' 'spark.driver.memory=1g' '--conf' 'spark.yarn.tags=livy-batch-0-D5xui8Hr' '--conf' 'spark.yarn.dist.files=' '--conf' 'spark.yarn.submit.waitAppCompletion=false' '--conf' 'spark.submit.deployMode=cluster' '--conf' 'spark.master=yarn' '--conf' 'spark.executor.cores=1' '--queue' 'default' 'hdfs:///griffin/griffin-measure.jar' '{
  "spark" : {
    "log.level" : "WARN",
    "checkpoint.dir" : "hdfs:///griffin/checkpoint/streaming_job_test",
    "init.clear" : true,
    "batch.interval" : "30s",
    "process.interval" : "3m",
    "config" : {
      "spark.default.parallelism" : 4,
      "spark.task.maxFailures" : 5,
      "spark.streaming.kafkaMaxRatePerPartition" : 1000,
      "spark.streaming.concurrentJobs" : 4,
      "spark.yarn.maxAppAttempts" : 5,
      "spark.yarn.am.attemptFailuresValidityInterval" : "1h",
      "spark.yarn.max.executor.failures" : 120,
      "spark.yarn.executor.failuresValidityInterval" : "1h",
      "spark.hadoop.fs.hdfs.impl.disable.cache" : true
    }
  },
  "persist" : [ {
    "type" : "log",
    "config" : {
      "max.log.lines" : 2
    }
  }, {
    "type" : "hdfs",
    "config" : {
      "path" : "hdfs:///griffin/persist"
    }
  }, {
    "type" : "http",
    "config" : {
      "method" : "post",
"livy.log" 1235 lines, 38645 characters
        },
        "topics" : "target",
        "key.type" : "java.lang.String",
        "value.type" : "java.lang.String"
      },
      "pre.proc" : [ {
        "id" : 459,
        "name" : "${this}",
        "rule" : "from_json",
        "dsl.type" : "df-opr",
        "details" : {
          "df.name" : "${this}"
        }
      } ]
    } ],
    "cache" : {
      "file.path" : "hdfs:///griffin/streaming/dump/target",
      "info.path" : "target",
      "ready.time.interval" : "10s",
      "ready.time.delay" : "0",
      "time.range" : [ "-1m", "0" ]
    }
  } ],
  "evaluate.rule" : {
    "id" : 452,
    "rules" : [ {
      "id" : 453,
      "rule" : "src.id = tgt.id AND src.name = tgt.name AND src.color = tgt.color AND src.time = tgt.time",
      "name" : "accu",
      "dsl.type" : "griffin-dsl",
      "dq.type" : "ACCURACY",
      "details" : {
        "source" : "src",
        "target" : "tgt"
      }
    } ]
  },
  "measure.type" : "griffin"
}' 'raw,raw'
18/05/10 07:07:10 INFO BatchSessionManager: Registering new session 7
18/05/10 08:08:03 INFO BatchSession: Stopping BatchSession 7...
18/05/10 08:08:03 INFO YarnClientImpl: Killed application application_1525765465898_0012
18/05/10 08:08:03 INFO BatchSession: Stopped BatchSession 7.