JAVA大数据的第五十七天——Hadoop中求每个学生的总分

JAVA大数据的第五十七天——Hadoop中求每个学生的总分

一、Student

package nj.zb.kb15.demo3;
//ctrl + 点击  进入
//ctrl + T 查看方法
//F4右侧展现方法
//查看当前类方法结构ctrl +alt +h
import org.apache.hadoop.io.WritableComparable;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

public class Student implements WritableComparable {
    private  long stuId;     //蛇形命名法 stu_id
    private  String stuName;
    private  int score;

    public Student() {
    }

    public Student(long stuId, String stuName, int score) {
        this.stuId = stuId;
        this.stuName = stuName;
        this.score = score;
    }

    @Override
    public String toString() {
        return "Student{" +
                "stuId=" + stuId +
                ", stuName='" + stuName + ''' +
                ", score=" + score +
                '}';
    }

    public long getStuId() {
        return stuId;
    }

    public void setStuId(long stuId) {
        this.stuId = stuId;
    }

    public String getStuName() {
        return stuName;
    }

    public void setStuName(String stuName) {
        this.stuName = stuName;
    }

    public int getScore() {
        return score;
    }

    public void setScore(int score) {
        this.score = score;
    }

    @Override
    public int compareTo(Student o) {
      // if (o.stuName==this.stuName && o.stuId==this.stuId)
     //    return 1;
        return 0;
    }

    @Override
    public void write(DataOutput dataOutput) throws IOException {
        dataOutput.writeLong(stuId);
        dataOutput.writeUTF(stuName);
        dataOutput.writeInt(score);
    }

    @Override
    public void readFields(DataInput dataInput) throws IOException {
        this.stuId=dataInput.readLong();
        this.stuName=dataInput.readUTF();
        this.score=dataInput.readInt();
    }
}

二、StudentScoreMapper

package nj.zb.kb15.demo3;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

public class StudentScoreMapper extends Mapper {
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        System.out.println(key.get()+" "+value);
        String[] split=value.toString().split(",");
        LongWritable stuid=new LongWritable(Long.parseLong(split[0]));
        Student student=new Student(stuid.get(),split[1],Integer.parseInt(split[3]));
        context.write(stuid,student);  //输出结果<1,student[1,名字，成绩]
    }
}

三、StudentScoreReducer

package nj.zb.kb15.demo3;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;


public class StudentScoreReducer extends Reducer {

   Student student=new Student();
    @Override
    protected  void reduce(LongWritable key, Iterable values, Context context) throws  IOException,InterruptedException{
        int sum=0;
        String  stuName="";
        for(Student stu:values){
            if(stuName.equals(""))
            stuName=stu.getStuName();
            sum+=stu.getScore();
        }
        student.setStuName(stuName);
        student.setStuId(key.get());
        student.setScore(sum);

        System.out.println(key.get()+"总成绩："+sum);
        context.write(key,student);
    }
}

四、StudentScoreDriver

package nj.zb.kb15.demo3;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

public class StudentScoreDriver {
    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        Configuration configuration=new Configuration();
        Job job=Job.getInstance(configuration);

        job.setJarByClass(StudentScoreDriver.class);

        job.setMapperClass(StudentScoreMapper.class);
        job.setMapOutputKeyClass(LongWritable.class);
        job.setMapOutputValueClass(Student.class);

        job.setReducerClass(StudentScoreReducer.class);
        job.setOutputKeyClass(LongWritable.class);
        job.setOutputValueClass(Student.class);

        FileInputFormat.setInputPaths(job,new Path("D:\test_20211026\in\demo3\stuscore.csv"));

        Path path=new Path("D:\test_20211026\in\out3");
        FileOutputFormat.setOutputPath(job,path);
        FileSystem fs=FileSystem.get(path.toUri(),configuration);

        if(fs.exists(path)){
            fs.delete(path,true);
        }
        job.waitForCompletion(true);
    }
}

五、运行结果

1   Student{stuId=1, stuName='鸿飞', score=174}
2  Student{stuId=2, stuName='鹏海', score=177}
3  Student{stuId=3, stuName='铭晨', score=180}
4  Student{stuId=4, stuName='霑千昌', score=183}
5  Student{stuId=5, stuName='智宇', score=186}
6  Student{stuId=6, stuName='景彰', score=189}
7  Student{stuId=7, stuName='辰铭', score=192}
8  Student{stuId=8, stuName='曜灿', score=195}
9  Student{stuId=9, stuName='昊苍', score=198}
10 Student{stuId=10, stuName='子昂', score=201}
11 Student{stuId=11, stuName='景行', score=204}
12 Student{stuId=12, stuName='昆皓', score=207}
13 Student{stuId=13, stuName='文昂', score=210}
14 Student{stuId=14, stuName='昊苍', score=164}
15 Student{stuId=15, stuName='德泽', score=167}
16 Student{stuId=16, stuName='鸿远', score=170}
17 Student{stuId=17, stuName='昌燎', score=173}
18 Student{stuId=18, stuName='昌翰', score=176}
19 Student{stuId=19, stuName='鸿振', score=179}
20 Student{stuId=20, stuName='鸿卓', score=182}
21 Student{stuId=21, stuName='浩初', score=185}
22 Student{stuId=22, stuName='运鹏', score=188}
23 Student{stuId=23, stuName='新曦', score=191}
24 Student{stuId=24, stuName='智阳', score=194}
25 Student{stuId=25, stuName='杨伟', score=197}