Hadoop學習筆記（八）---內置數據類型與自定義數據類型

例如我們要計算下面數據的同一電話號碼（5,6,7,8位置）相同位置數據相加結果：

1363157985066 13726230503 00-FD-07-A4-72-B8:CMCC 120.196.100.82 24 27 2481 24681 200
1363157985066 13726230503 00-FD-07-A4-72-B8:CMCC 120.196.100.82 24 27 2481 24681 200
1363157985066 13726230503 00-FD-07-A4-72-B8:CMCC 120.196.100.82 24 27 2481 24681 200
1363157985066 13726230503 00-FD-07-A4-72-B8:CMCC 120.196.100.82 24 27 2481 24681 200
1363157985066 13726230503 00-FD-07-A4-72-B8:CMCC 120.196.100.82 24 27 2481 24681 200

我們來寫一個自定義類，但是必須實現Writeable接口：

package cn.edu.bjut.model;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

import org.apache.hadoop.io.Writable;

public class DataWriteable implements Writable {

    private long a, b, c, d;

    public DataWriteable() {
        super();
    }

    public DataWriteable(String a, String b, String c, String d) {
        super();
        this.a = Long.parseLong(a);
        this.b = Long.parseLong(b);
        this.c = Long.parseLong(c);
        this.d = Long.parseLong(d);
    }

    @Override
    public void readFields(DataInput in) throws IOException {

        this.a = in.readLong();
        this.b = in.readLong();
        this.c = in.readLong();
        this.d = in.readLong();

    }

    @Override
    public void write(DataOutput out) throws IOException {

        out.writeLong(a);
        out.writeLong(b);
        out.writeLong(c);
        out.writeLong(d);

    }

    @Override
    public String toString() {
        return a + "\t" + b + "\t" + d + "\t" + d;
    }

    public long getA() {
        return a;
    }

    public void setA(long a) {
        this.a = a;
    }

    public long getB() {
        return b;
    }

    public void setB(long b) {
        this.b = b;
    }

    public long getC() {
        return c;
    }

    public void setC(long c) {
        this.c = c;
    }

    public long getD() {
        return d;
    }

    public void setD(long d) {
        this.d = d;
    }

}

主方法這麼寫：

package cn.edu.bjut.model;
import java.io.IOException;
import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;


public class NumCount {

    static final String INPUT_DIR = "hdfs://172.21.15.189:9000/input";
    static final String OUTPUT_DIR = "hdfs://172.21.15.189:9000/output";

    public static void main(String[] args) throws Exception {

        Configuration conf = new Configuration();

        Path path = new Path(OUTPUT_DIR);

        FileSystem fileSystem = FileSystem.get(new URI(OUTPUT_DIR), conf);

        if(fileSystem.exists(path)) {

            fileSystem.delete(path, true);

        }

        Job job = new Job(conf, "NumCount");

        FileInputFormat.setInputPaths(job, INPUT_DIR); //設置輸入路徑
        FileOutputFormat.setOutputPath(job, path);  //設置輸出路徑

        job.setMapperClass(MyMapper.class); //設置自定義的mapper類

        job.setReducerClass(MyReducer.class);  //設置自定義的reduce類

        job.setOutputKeyClass(Text.class);  //設置輸出的key的類型

        job.setOutputValueClass(DataWriteable.class);  //設置輸出的value類型

        job.waitForCompletion(true);  //開始執行

    }


    /**
     * 自定義的map類
     * @author Gary
     *
     */
    static class MyMapper extends Mapper<LongWritable, Text, Text, DataWriteable> {

        @Override
        protected void map(LongWritable key, Text value,
                Mapper<LongWritable, Text, Text, DataWriteable>.Context context)
                throws IOException, InterruptedException {

            String[] all = value.toString().split(" ");

            DataWriteable dataWriteable = new DataWriteable(all[4], all[5], all[6], all[7]);

            context.write(new Text(all[1]), dataWriteable);
        }



    }

    /**
     * 自定義的reduce類
     * @author Gary
     *
     */
    static class MyReducer extends Reducer<Text, DataWriteable, Text, DataWriteable> {

        @Override
        protected void reduce(Text key, Iterable<DataWriteable> values,
                Reducer<Text, DataWriteable, Text, DataWriteable>.Context context)
                throws IOException, InterruptedException {

            long a = 0L, b = 0L, c = 0L, d = 0L;

            for(DataWriteable dataWriteable : values) {

                a += dataWriteable.getA();
                b += dataWriteable.getB();
                c += dataWriteable.getC();
                d += dataWriteable.getD();

            }

            DataWriteable dataWriteable = new DataWriteable(a+"", b+"", c+"", d+"");

            context.write(key, dataWriteable);

        }



    }

}

運行程序最終結果爲：

13726230503 120 135 123405  123405