[root@linux01 install]# tar -zxf flume-ng-1.6.0-cdh5.14.0.tar.gz -C ../soft [root@linux01 soft]# mv apache-flume-1.6.0-cdh5.14.0-bin/ flume160 [root@linux01 soft]# cd flume160/ [root@linux01 flume160]# cd conf/ [root@linux01 conf]# cp flume-env.sh.template flume-env.sh [root@linux01 conf]# vi ./flume-env.sh 22 export JAVA_HOME=/opt/soft/jdk180 25 # export JAVA_OPTS="-Xms3000m -Xmx3000m -Dcom.sun.management.jmxremote" [root@linux01 conf]# yum install -y nc [root@linux01 conf]# nc -lk 44444 [root@linux01 conf]# yum install telnet-server [root@linux01 conf]# yum install telnet.* [root@linux01 conf]# telnet localhost 44444 println("--------------------------------------------") [root@linux01 kb15conf]# vi ./netcat-flume-logger.conf a1.sources=r1 a1.channels=c1 a1.sinks=k1 a1.sources.r1.type=netcat a1.sources.r1.bind=localhost a1.sources.r1.port=44444 a1.channels.c1.type=memory a1.channels.c1.capacity=1000 a1.channels.c1.transactionCapacity=100 a1.sinks.k1.type=logger a1.sources.r1.channels=c1 a1.sinks.k1.channel=c1 [root@linux01 flume160]# ./bin/flume-ng agent --name a1 --conf ./conf/ --conf-file ./conf/kb15conf/netcat-flume-logger.conf -Dflume.root.logger=INFO,console(先) [root@linux01 conf]# telnet localhost 44444(后) println("--------------------------------------------") logger [root@linux01 flume160]#./bin/flume-ng agent --name a2 --conf ./conf/ --conf-file ./conf/kb15conf/file-flume-logger.conf -Dflume.root.logger=INFO,console(先) [root@linux01 kb15conf]# echo hello spark >> ./flumefile.log(后) [root@linux01 kb15conf]# echo hello flume >> ./flumefile.log(后) [root@linux01 kb15conf]# vi ./file-flume-hdfs.conf a2.sources=r1 a2.channels=c1 a2.sinks=k1 a2.sources.r1.type=exec a2.sources.r1.command=tail -f /opt/soft/flume160/conf/kb15conf/flumefile.log a2.channels.c1.type=memory a2.channels.c1.capacity=1000 a2.channels.c1.transactionCapacity=100 a2.sinks.k1.type=hdfs a2.sinks.k1.hdfs.fileType=DataStream a2.sinks.k1.hdfs.filePrefix=flumetohdfs a2.sinks.k1.hdfs.fileSuffix=.txt a2.sinks.k1.hdfs.path=hdfs://192.168.111.131:9000/kb15file a2.sources.r1.channels=c1 a2.sinks.k1.channel=c1 上传到hdfs [root@linux01 kb15conf]# hdfs dfsadmin -safemode leave [root@linux01 flume160]# ./bin/flume-ng agent --name a2 --conf ./conf/ --conf-file ./conf/kb15conf/file-flume-hdfs.conf -Dflume.root.logger=INFO,console(先) [root@linux01 kb15conf]# echo hello spark >> ./flumefile.log(后) [root@linux01 kb15conf]# echo hello flume >> ./flumefile.log(后)
欢迎分享,转载请注明来源:内存溢出
评论列表(0条)