JAVA大数据的第五十七天——Hadoop中求每个学生三门科目的最高分

一、TopStudent

package nj.zb.kb15.demo5;

import org.apache.hadoop.io.WritableComparable;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

//每个人三门科目的最高分
public class TopStudent  implements WritableComparable {
    private  long stu_id;
    private  String stu_name;
    private  String sub_name;
    private  int score;


    public TopStudent() {
    }

    public TopStudent(long stu_id, String stu_name, int score, String sub_name) {
        this.stu_id = stu_id;
        this.stu_name = stu_name;
        this.score = score;
        this.sub_name = sub_name;
    }

    @Override
    public String toString() {
        return "TopStudent{" +
                "stu_id=" + stu_id +
                ", stu_name='" + stu_name + ''' +
                ", score=" + score +
                ", sub_name='" + sub_name + ''' +
                '}';
    }

    public long getStu_id() {
        return stu_id;
    }

    public void setStu_id(long stu_id) {
        this.stu_id = stu_id;
    }

    public String getStu_name() {
        return stu_name;
    }

    public void setStu_name(String stu_name) {
        this.stu_name = stu_name;
    }

    public int getScore() {
        return score;
    }

    public void setScore(int score) {
        this.score = score;
    }

    public String getSub_name() {
        return sub_name;
    }

    public void setSub_name(String sub_name) {
        this.sub_name = sub_name;
    }

    @Override
    public int compareTo(TopStudent o) {
        return 0;
    }

    @Override
    public void write(DataOutput dataOutput) throws IOException {
        dataOutput.writeLong(stu_id);
        dataOutput.writeUTF(stu_name);
        dataOutput.writeUTF(sub_name);
        dataOutput.writeInt(score);


    }

    @Override
    public void readFields(DataInput dataInput) throws IOException {
            this.stu_id=dataInput.readLong();
            this.stu_name=dataInput.readUTF();
           this.sub_name=dataInput.readUTF();
            this.score=dataInput.readInt();


    }
}

二、TopStudentMapper

package nj.zb.kb15.demo5;


import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

public class TopStudentMapper extends Mapper {
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        System.out.println(key.get()+" "+value);
        String[] split=value.toString().split(",");
        LongWritable stuId=new LongWritable(Long.parseLong(split[0]));
        String sub_name=split[2];
        TopStudent topStudent=new TopStudent(stuId.get(),split[1],Integer.parseInt(split[3]),sub_name);
        context.write(stuId,topStudent);
    }
}

三、TopStudentReducer

package nj.zb.kb15.demo5;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

public class TopStudentReducer extends Reducer {
    TopStudent topStudent=new TopStudent();
    @Override
    protected void reduce(LongWritable key, Iterable values, Context context) throws IOException, InterruptedException {
        String subName="";
        String stuName="";
        int max=0;
        for(TopStudent stu:values){
             if(max 
四、TopStudentDtriver
 
package nj.zb.kb15.demo5;


import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

public class TopStudentDriver {
    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        Configuration configuration=new Configuration();
        Job job=Job.getInstance(configuration);

        job.setJarByClass(TopStudentDriver.class);

        job.setMapperClass(TopStudentMapper.class);
        job.setMapOutputKeyClass(LongWritable.class);
        job.setMapOutputValueClass(TopStudent.class);

        job.setReducerClass(TopStudentReducer.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(TopStudent.class);

        FileInputFormat.setInputPaths(job,new Path("D:\test_20211026\in\demo3\stuscore.csv"));

        Path path=new Path("D:\test_20211026\in\out6");
        FileOutputFormat.setOutputPath(job,path);
        FileSystem fs=FileSystem.get(path.toUri(),configuration);

        if(fs.exists(path)){
            fs.delete(path,true);
        }
        job.waitForCompletion(true);
    }
}
 
五、运行结果
 
1  TopStudent{stu_id=1, stu_name='鸿飞', score=84, sub_name='语文'}
2  TopStudent{stu_id=2, stu_name='鹏海', score=85, sub_name='语文'}
3  TopStudent{stu_id=3, stu_name='铭晨', score=86, sub_name='语文'}
4  TopStudent{stu_id=4, stu_name='霑千昌', score=87, sub_name='语文'}
5  TopStudent{stu_id=5, stu_name='智宇', score=88, sub_name='语文'}
6  TopStudent{stu_id=6, stu_name='景彰', score=89, sub_name='语文'}
7  TopStudent{stu_id=7, stu_name='辰铭', score=90, sub_name='语文'}
8  TopStudent{stu_id=8, stu_name='曜灿', score=91, sub_name='语文'}
9  TopStudent{stu_id=9, stu_name='昊苍', score=92, sub_name='语文'}
10 TopStudent{stu_id=10, stu_name='子昂', score=93, sub_name='语文'}
11 TopStudent{stu_id=11, stu_name='景行', score=94, sub_name='语文'}
12 TopStudent{stu_id=12, stu_name='昆皓', score=95, sub_name='语文'}
13 TopStudent{stu_id=13, stu_name='文昂', score=96, sub_name='语文'}
14 TopStudent{stu_id=14, stu_name='昊苍', score=80, sub_name='数学'}
15 TopStudent{stu_id=15, stu_name='德泽', score=81, sub_name='数学'}
16 TopStudent{stu_id=16, stu_name='鸿远', score=82, sub_name='数学'}
17 TopStudent{stu_id=17, stu_name='昌燎', score=83, sub_name='数学'}
18 TopStudent{stu_id=18, stu_name='昌翰', score=84, sub_name='数学'}
19 TopStudent{stu_id=19, stu_name='鸿振', score=85, sub_name='数学'}
20 TopStudent{stu_id=20, stu_name='鸿卓', score=86, sub_name='数学'}
21 TopStudent{stu_id=21, stu_name='浩初', score=87, sub_name='数学'}
22 TopStudent{stu_id=22, stu_name='运鹏', score=88, sub_name='数学'}
23 TopStudent{stu_id=23, stu_name='新曦', score=89, sub_name='数学'}
24 TopStudent{stu_id=24, stu_name='智阳', score=90, sub_name='数学'}
25 TopStudent{stu_id=25, stu_name='杨伟', score=91, sub_name='数学'}
					
										


					

						
欢迎分享，转载请注明来源：内存溢出
原文地址: http://outofmemory.cn/zaji/4969591.html