[FLUME-3232] some files gibberish

• Type: Bug
• Status: Open
• Priority: Blocker
• Resolution: Unresolved
• Affects Version/s: 1.8.0
• Component/s: File Channel, Sinks+Sources

Environment

Ubuntu 5.4.0-6ubuntu1~16.04.9

• Assignee: Unassigned
• Reporter: lee hj
• Created: 27/Apr/18 10:57
• Updated: 04/Aug/18 22:21

Description

my conf file like this :

 

conf1:

a1.sources = r1
a1.sinks = k1 k2 k3
a1.channels = c1

a1.sinkgroups = g1
a1.sinkgroups.g1.sinks = k1 k2 k3
a1.sinkgroups.g1.processor.type =load_balance
a1.sinkgroups.g1.processor.backoff = true
a1.sinkgroups.g1.processor.selector =round_robin

a1.sources.r1.type = TAILDIR
a1.sources.r1.channels = c1
a1.sources.r1.positionFile = /data/flume/position.json
a1.sources.r1.filegroups = f1
a1.sources.r1.filegroups.f1.parentDir = /data/platform/pfdata/advert/appsflyer
a1.sources.r1.filegroups.f1.filePattern = .*
a1.sources.r1.cachePatternMatching = false
a1.sources.r1.skipToEnd = false
a1.sources.r1.batchSize = 1000
a1.sources.r1.backoffSleepIncrement = 1000
a1.sources.r1.maxBackoffSleep = 5000
a1.sources.r1.fileHeader = true
a1.sources.r1.interceptors = i1 i2
a1.sources.r1.interceptors.i1.type = static
a1.sources.r1.interceptors.i1.key = official
a1.sources.r1.interceptors.i1.value = true
a1.sources.r1.interceptors.i2.type = static
a1.sources.r1.interceptors.i2.key = game_cd
a1.sources.r1.interceptors.i2.value = none

a1.sinks.k1.type = avro
a1.sinks.k1.channel = c1
a1.sinks.k1.hostname = 10.0.4.239
a1.sinks.k1.port = 44445
a1.sinks.k1.batch-size = 1000

a1.channels.c1.type = file
a1.channels.c1.checkpointDir = /data/flume/checkpoint
a1.channels.c1.dataDirs = /data/flume/data
a1.channels.c1.keep-alive = 8
a1.channels.c1.capacity = 1000000
a1.channels.c1.transactionCapacity = 10000

conf2:
tap4fun_log_agent.sources.r1.type = avro
tap4fun_log_agent.sources.r1.bind = 10.0.4.239
tap4fun_log_agent.sources.r1.port = 44444
tap4fun_log_agent.sources.r1.channels = c1 c2 c3
tap4fun_log_agent.sources.r1.interceptors = i1
tap4fun_log_agent.sources.r1.interceptors.i1.type = static
tap4fun_log_agent.sources.r1.interceptors.i1.key = host
tap4fun_log_agent.sources.r1.interceptors.i1.value = collector01

a1.sources.r1.type = avro
a1.sources.r1.bind = 10.0.4.239
a1.sources.r1.port = 44444
a1.sources.r1.channels = c1 c2 c3
a1.sources.r1.interceptors = i1
a1.sources.r1.interceptors.i1.type = static
a1.sources.r1.interceptors.i1.key = host
a1.sources.r1.interceptors.i1.value = collector01

a1.channels.c1.type = file
a1.channels.c1.checkpointDir = /home/developer/app/flume/file-channel/checkpoint
a1.channels.c1.dataDirs = /home/developer/app/flume/file-channel/data
a1.channels.c1.keep-alive = 8
a1.channels.c1.capacity = 1000000
a1.channels.c1.transactionCapacity = 10000

a1.sinks.k1.type = hdfs
a1.sinks.k1.channel = c1
a1.sinks.k1.hdfs.path = hdfs://nm-cluster/apps/hive/warehouse/stg.db/%

{file}
a1.sinks.k1.hdfs.filePrefix = data/%{file}

%Y-%m-%d%

{host}

a1.sinks.k1.hdfs.inUsePrefix = tmp/
a1.sinks.k1.hdfs.useLocalTimeStamp = true
a1.sinks.k1.hdfs.fileType = DataStream
a1.sinks.k1.hdfs.rollInterval = 600
a1.sinks.k1.hdfs.rollSize = 104857600
a1.sinks.k1.hdfs.rollCount = 0
a1.sinks.k1.hdfs.batchSize = 1000
a1.sinks.k1.hdfs.callTimeout = 50000
a1.sinks.k1.hdfs.idleTimeout = 800

taildirsource+file+avro sink=>avro source+file+hdfs sink
my hdfs get some garbled files,Most of the files are normal
.would you give me some help?