2021.12.28kafkaStream

2021.12.28kafkaStream

UserFriendsStream

import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.common.serialization.Serdes;
import org.apache.kafka.streams.*;
import org.apache.kafka.streams.errors.StreamsException;
import org.apache.kafka.streams.kstream.KStream;

import java.util.ArrayList;
import java.util.Properties;
import java.util.concurrent.CountDownLatch;


public class UserFriendsStream {
    public static void main(String[] args) {
        Properties prop = new Properties();
        prop.put(StreamsConfig.APPLICATION_ID_CONFIG,"userFriend");
        prop.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG,"192.168.111.131:9092");
        prop.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG,3000);
        prop.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG,"false");
        //earliest latest none
        prop.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG,"earliest");
        prop.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
        prop.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG,Serdes.String().getClass());


        StreamsBuilder builder = new StreamsBuilder();
        KStream user_friends_rows = builder.stream("user_friends_rows")
                .flatMap((key, value) -> { //3197468391,1346449342 387324416 4226080662
                    ArrayList> list = new ArrayList<>();
                    String[] fields = value.toString().split(","); //[3197468391,1346449342 387324416 4226080662]
                    if (fields.length == 2) {
                        String[] friends = fields[1].split("\s+");  //[1346449342 387324416 4226080662]
                        String user = fields[0];                           // 3197468391
                        if (user.trim().length() > 0) {  //  \s匹配任意空白字符
                            for (String friend :
                                    friends) {
                                System.out.println(user + "," + friend);
                                KeyValue keyValue = new KeyValue<>(null, user + "," + friend);
                                list.add(keyValue);

                            }
                        }
                    }
                    return list;


                });

        user_friends_rows.to("user_friends");

        //构建 Topology
        Topology topo = builder.build();
        final KafkaStreams streams = new KafkaStreams(topo, prop);

        final CountDownLatch latch = new CountDownLatch(1);

        Runtime.getRuntime().addShutdownHook(new Thread("stream"){
            @Override
            public void run() {
                streams.close();
                latch.countDown();
            }
        });



        try {
            streams.start();
            latch.countDown();
        }catch (IllegalStateException e){
            e.printStackTrace();
        }catch (StreamsException e){
            e.printStackTrace();
    }

//        System.exit(0);


    }
}

 EventAttendStream

import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.common.serialization.Serdes;
import org.apache.kafka.streams.*;
import org.apache.kafka.streams.errors.StreamsException;
import org.apache.kafka.streams.kstream.KStream;

import java.util.ArrayList;
import java.util.Properties;
import java.util.concurrent.CountDownLatch;
//2021.12.28
public class EventAttendStream {
    public static void main(String[] args) {
        Properties prop = new Properties();
        prop.put(StreamsConfig.APPLICATION_ID_CONFIG,"eventattend");
        prop.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG,"192.168.111.131:9092");
        prop.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
        prop.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG,Serdes.String().getClass());
        prop.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG,3000);
        prop.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG,"false"); //自动提交
        //earliest latest none
        prop.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG,"earliest");


        StreamsBuilder builder = new StreamsBuilder();
        KStream ear = builder.stream("event_attendess_row"); //流
        ear.flatMap((key,value)->{
            ArrayList>list=new ArrayList<>();
            String[] fields = value.toString().split(",");
            String eventid = fields[0];
            //yes
            if (fields.length>=2 && fields[1].trim().length()>0){
                String[]yes=fields[1].trim().split("\s+");
                for (String y:
                yes){
                    System.out.println(eventid+","+y+",yes");
                    KeyValue yesKeyValue = new KeyValue<>(null, eventid + "," + y + ",yes");
                    list.add(yesKeyValue);
                }

            }

            //maybe
            if (fields.length>=3 && fields[2].trim().length()>0){
                String[]maybe=fields[2].trim().split("\s+");
                for (String mb:
                        maybe){
                    System.out.println(eventid+","+mb+",maybe");
                    KeyValue mbKeyValue = new KeyValue<>(null, eventid + "," + mb + ",maybe");
                    list.add(mbKeyValue);
                }

            }

            //invited
            if (fields.length>=4 && fields[3].trim().length()>0){
                String[]invited=fields[3].trim().split("\s+");
                for (String i:
                        invited){
                    System.out.println(eventid+","+i+",invited");
                    KeyValue invitedKeyValue = new KeyValue<>(null, eventid + "," + i + ",invited");
                    list.add(invitedKeyValue);
                }

            }
            //no
            if (fields.length>=5 && fields[4].trim().length()>0){
                String[]nos=fields[4].trim().split("\s+");
                for (String no:
                        nos){
                    System.out.println(eventid+","+no+",no");
                    KeyValue noKeyValue = new KeyValue<>(null, eventid + "," + no + ",no");
                    list.add(noKeyValue);
                }

            }



            return list;


        }).to("event_attendess");

        Topology topo = builder.build();
        final KafkaStreams streams = new KafkaStreams(topo, prop);
        final CountDownLatch latch = new CountDownLatch(1);

        Runtime.getRuntime().addShutdownHook(new Thread("stream"){
            @Override
            public void run() {
                streams.close();
                latch.countDown();
            }
        });

        try {
            streams.start();
            latch.await();
        } catch (IllegalStateException e) {
                e.printStackTrace();
        } catch (StreamsException e) {
            e.printStackTrace();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }

        System.exit(0);


    }

}

 MyStreamTest

import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.common.serialization.Serdes;
import org.apache.kafka.streams.KafkaStreams;
import org.apache.kafka.streams.StreamsBuilder;
import org.apache.kafka.streams.StreamsConfig;
import org.apache.kafka.streams.Topology;
import org.apache.kafka.streams.errors.StreamsException;
import org.apache.kafka.streams.kstream.KStream;

import java.util.Properties;
import java.util.concurrent.CountDownLatch;

public class MyStreamTest {
    public static void main(String[] args) {
        Properties prop = new Properties();
        prop.put(StreamsConfig.APPLICATION_ID_CONFIG, "myStream");
        prop.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.111.131:9092");
        prop.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 3000);
        prop.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
        //earliest latest none
        prop.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
        prop.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
        prop.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());

        StreamsBuilder builder = new StreamsBuilder();
        KStream in = builder.stream("mystreamin");
        in.to("mystreamout");


        Topology topo = builder.build();
        final KafkaStreams streams = new KafkaStreams(topo, prop);
        final CountDownLatch latch = new CountDownLatch(1);

        Runtime.getRuntime().addShutdownHook(new Thread("stream"){
            @Override
            public void run() {
                streams.close();
                latch.countDown();
            }
        });

        try {
            streams.start();
            latch.await();
        } catch (IllegalStateException e) {
            e.printStackTrace();
        } catch (StreamsException e) {
            e.printStackTrace();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }

        System.exit(0);


    }

}