分布式flume日志收集系统

source 采用exec方式代替，在两台机器上部署
a1.sources=s1
a1.channels=c1
a1.sinks=s1

#source
a1.sources.s1.type=exec
a1.sources.s1.command=tail -f /home/hadoop/data.log

#sink

a1.sinks.s1.type=avro
a1.sinks.s1.hostname=centos-1
a1.sinks.s1.port=44444

#channel
a1.channels.c1.type=memory
a1.channels.c1.capacity=1000
a1.channels.c1.transactionCapacity = 1000

a1.sources.s1.channels=c1
a1.sinks.s1.channel=c1

第二类agent
a1.sources = r1
a1.sinks = k1
a1.channels = c1

# Describe/configure the source
a1.sources.r1.type = avro
a1.sources.r1.bind = centos-1
a1.sources.r1.port = 44444
a1.sources.r1.interceptors = i1
a1.sources.r1.interceptors.i1.type = timestamp  拦截器给event增加时间

# Describe the sink
a1.sinks.k1.type = hdfs
a1.sinks.k1.hdfs.path = hdfs://centos-1:8020/flume/events/%Y/%m/%d/
a1.sinks.k1.hdfs.filePrefix = events-
a1.sinks.k1.hdfs.rollSize=1024
a1.sinks.k1.hdfs.rollInterval=0
a1.sinks.k1.hdfs.rollCount=0

# Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100

# Bind the source and sink to the channel
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1