Maven工程的MapReduce程序4---MapReduce序列化、分區實驗

MapReduce序列化、分區實驗

有一張員工表emp.csv,內容如下：

SAL:爲員工工資

7369,SMITH,CLERK,7902,1980/12/17,800,,20
7499,ALLEN,SALESMAN,7698,1981/2/20,1600,300,30
7521,WARD,SALESMAN,7698,1981/2/22,1250,500,30
7566,JONES,MANAGER,7839,1981/4/2,2975,,20
7654,MARTIN,SALESMAN,7698,1981/9/28,1250,1400,30
7698,BLAKE,MANAGER,7839,1981/5/1,2850,,30
7782,CLARK,MANAGER,7839,1981/6/9,2450,,10
7788,SCOTT,ANALYST,7566,1987/4/19,3000,,20
7839,KING,PRESIDENT,,1981/11/17,5000,,10
7844,TURNER,SALESMAN,7698,1981/9/8,1500,0,30
7876,ADAMS,CLERK,7788,1987/5/23,1100,,20
7900,JAMES,CLERK,7698,1981/12/3,950,,30
7902,FORD,ANALYST,7566,1981/12/3,3000,,20
7934,MILLER,CLERK,7782,1982/1/23,1300,,10

根據如上emp.csv表，假設：

薪資<1500，爲低薪，

薪資>=1500，而且薪資<3000爲中薪，

薪資>=3000，爲高薪。

問題：編寫程序實現將對員工數據按低薪、中薪、高薪進行分區存儲，輸出到三個文件。

要求：職工信息採用一個獨立的類存放，並且實現Hadoop序列化。

 

本實驗是在案例四的基礎上進行分析：

由以上分析，一共有5個類

新建Maven工程，配置好pom.xml（參考案例二），建立相應的5各類。

 

參考代碼：

Employee.java

package com.myPatition2;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

import org.apache.hadoop.io.Writable;

//定義Employee類實現序列化接口
public class Employee implements Writable{
	
	//字段名 EMPNO, ENAME,    JOB,   MGR,   HIREDATE,  SAL, COMM, DEPTNO
	//數據類型：Int，Char，          Char  ， Int，     Date  ，       Int   Int，  Int
	//數據: 7654, MARTIN, SALESMAN, 7698, 1981/9/28, 1250, 1400, 30
	
	//由以上定義變量
	private int empno;
	private String ename;
	private String job;
	private int mgr;
	private String hiredate;
	private int sal;
	private int comm;//獎金
	private int deptno;
	
	

	@Override
	public String toString() {
//		return "Employee [empno=" + empno + ", ename=" + ename + ", sal=" + sal + ", deptno=" + deptno + "]";
		return empno+","+ename+","+job+","+mgr+","+hiredate+","+sal+","+comm+","+deptno;
	}
	//序列化方法：將java對象轉化爲可跨機器傳輸數據流（二進制串/字節）的一種技術
	public void write(DataOutput out) throws IOException {
		out.writeInt(this.empno);
		out.writeUTF(this.ename);
		out.writeUTF(this.job);
		out.writeInt(this.mgr);
		out.writeUTF(this.hiredate);
		out.writeInt(this.sal);
		out.writeInt(this.comm);
		out.writeInt(this.deptno);
		
	}
	//反序列化方法：將可跨機器傳輸數據流（二進制串）轉化爲java對象的一種技術
	public void readFields(DataInput in) throws IOException {
		this.empno = in.readInt();
		this.ename = in.readUTF();
		this.job = in.readUTF();
		this.mgr = in.readInt();
		this.hiredate = in.readUTF();
		this.sal = in.readInt();
		this.comm = in.readInt();
		this.deptno = in.readInt();
	}
	//其他類通過set/get方法操作變量：Source-->Generator Getters and Setters
	public int getEmpno() {
		return empno;
	}
	public void setEmpno(int empno) {
		this.empno = empno;
	}
	public String getEname() {
		return ename;
	}
	public void setEname(String ename) {
		this.ename = ename;
	}
	public String getJob() {
		return job;
	}
	public void setJob(String job) {
		this.job = job;
	}
	public int getMgr() {
		return mgr;
	}
	public void setMgr(int mgr) {
		this.mgr = mgr;
	}
	public String getHiredate() {
		return hiredate;
	}
	public void setHiredate(String hiredate) {
		this.hiredate = hiredate;
	}
	public int getSal() {
		return sal;
	}
	public void setSal(int sal) {
		this.sal = sal;
	}
	public int getComm() {
		return comm;
	}
	public void setComm(int comm) {
		this.comm = comm;
	}
	public int getDeptno() {
		return deptno;
	}
	public void setDeptno(int deptno) {
		this.deptno = deptno;
	}
	
	

}

注意：Employee類 ，要重寫toString()方法，構造出Reduce所要的輸出。

 

SalaryTotalMapper

package com.myPatition2;

import java.io.IOException;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

public class SalaryTotalMapper extends Mapper< LongWritable, Text, NullWritable,  Employee> {

	@Override
	protected void map(LongWritable k1, Text v1,
			Context context)
			throws IOException, InterruptedException {
		//數據：7499,ALLEN,SALESMAN,7698,1981/2/20,1600,300,30
		String data = v1.toString();
		String[] words = data.split(",");
		
		//創建員工對象
		Employee emp = new Employee();
		//設置員工屬性
		emp.setEmpno(Integer.parseInt(words[0]));
		
		emp.setEname(words[1]);
		
		emp.setJob(words[2]);
		
		try {
			emp.setMgr(Integer.parseInt(words[3]));//可能爲空,加try...catch包圍
		} catch (NumberFormatException ex) {
			ex.printStackTrace();
		}
		
		emp.setHiredate(words[4]);
		
		emp.setSal(Integer.parseInt(words[5]));
		
		try {
			emp.setComm(Integer.parseInt(words[6]));//可能爲空
		} catch (NumberFormatException ex) {
			ex.printStackTrace();
		}
		
		emp.setDeptno(Integer.parseInt(words[7]));
		
		//取出部門號words[7]，將String轉換爲Int，Int轉換爲IntWritable對象，賦值爲k2
		NullWritable k2 = NullWritable.get();
		//取出工資words[5]，將String轉換爲Int，Int轉換爲IntWritable對象，賦值爲v2
		Employee v2 = emp;
		//輸出k2, v2
		context.write(k2, v2);
	}
	
}

 SalaryTotalReducer.java

package com.myPatition2;

import java.io.IOException;

import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
                                                                                 
public class SalaryTotalReducer extends Reducer<NullWritable,Employee,NullWritable,Text> {

	@Override
	protected void reduce(NullWritable k3, Iterable<Employee> v3,
			Context context) throws IOException, InterruptedException {
		String line=null;
		for (Employee v : v3) {
			line = v.toString();
			context.write(k3, new Text(line));
		}		
	}
}

SalaryTotalMain.java 

 

package com.myPatition2;


import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class SalaryTotalMain {
	public static void main(String[] args) throws Exception {
		//1. 創建一個job和任務入口(指定主類)
		Job job = Job.getInstance();
		job.setJarByClass(SalaryTotalMain.class);
		
		//2. 指定job的mapper和輸出的類型<k2 v2>
		job.setMapperClass(SalaryTotalMapper.class);
		job.setMapOutputKeyClass(NullWritable.class);
		job.setMapOutputValueClass(Employee.class);
		
		//這裏有變化：
		//指定任務的分區規則的類
		job.setPartitionerClass(SalaryTotalPartitioner.class);
		//指定建立幾個分區
		job.setNumReduceTasks(3);
		
		//3. 指定job的reducer和輸出的類型<k4  v4>
		job.setReducerClass(SalaryTotalReducer.class);
		job.setOutputKeyClass(NullWritable.class);
		job.setOutputValueClass(Text.class);
		
		//4. 指定job的輸入和輸出路徑
		FileInputFormat.setInputPaths(job, new Path(args[0]));
		FileOutputFormat.setOutputPath(job, new Path(args[1]));
		//5. 執行job
		job.waitForCompletion(true);

	}
}

SalaryTotalPartitioner.java 

package com.myPatition2;

import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Partitioner;
//                                    map-outputs:k2,v2-->NullWritable, Employee 
public class SalaryTotalPartitioner extends Partitioner<NullWritable, Employee>{

	@Override
	public int getPartition(NullWritable k2, Employee v2, int numPatition) {
		
		//如何分區: 每個部門放在一個分區
		if(v2.getSal() < 1500) {
			//放入1號分區中
			return 1%numPatition;// 1%3=1
		}else if(v2.getSal() >=1500 && v2.getSal() < 3000){
			//放入2號分區中
			return 2%numPatition;// 2%3=2
		}else {
			//放入3號分區中
			return 3%numPatition;// 3%3=0
		}		
	}
}

寫好代碼後，打成jar包，提交到hadoop去執行

查看輸出文件如下 ：除了_SUCCESS外，有三個輸出文件。

查看輸出結果，確實是按照薪水等級來存放的員工數據的，說明程序已正確。 

 

完成！ enjoy it！