MapReduce Unit Test

　　以前用java写MR程序总不习惯写单元测试，就是查错也只是在小规模数据上跑一下程序。昨天工作时，遇到一个bug，查了好久也查出来。估计是业务逻辑上的错误。后来没办法，只好写了个单元测试，一步步跟踪，瞬间找到问题所在。所以说，工作中还是要勤快些。

import static org.junit.Assert.assertEquals;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mrunit.mapreduce.MapDriver;
import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;
import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
import org.apache.hadoop.mrunit.types.Pair;
import org.junit.Before;
import org.junit.Test;
import com.wanda.predict.GenerateCustomerNatureFeature.NatureFeatureMappper;
import com.wanda.predict.GenerateCustomerNatureFeature.NatureReducer;
import com.wanda.predict.pojo.Settings;

/**
* MapReduce 单元测试的模板 , 依赖于junit环境(junit.jar)， mrunit.jar , mockito.jar
*
*/
public class MapperReducerUnitTest {
// 一些设置，与正常的mr程序一样，不过这里主要是加载一些信息。性能优化之类的就不要在单元测试里设置了。
Configuration conf = new Configuration();
//Map.class 的测试驱动类
MapDriver<LongWritable, Text, Text, Text> mapDriver;
//Reduce.class 的测试驱动类
ReduceDriver<Text, Text, Text, Text> reduceDriver;
//Map.calss 、 Reduce.class转接到一起的流程测试驱动
MapReduceDriver<LongWritable, Text, Text, Text, Text, Text> mapReduceDriver;

@Before
public void setUp() {

//测试mapreduce
NatureFeatureMappper mapper = new NatureFeatureMappper();
NatureReducer reducer = new NatureReducer();
//添加要测试的map类
mapDriver = MapDriver.newMapDriver(mapper);
//添加要测试的reduce类
reduceDriver = ReduceDriver.newReduceDriver(reducer);
//添加map类和reduce类
mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);

//测试配置参数
conf.setInt(Settings.TestDataSize.getName(), 1);
conf.setInt(Settings.TrainDataSize.getName(), 6);
//driver之间是独立的，谁用到谁就设置conf
reduceDriver.setConfiguration(conf);
mapReduceDriver.setConfiguration(conf);
}

@Test
public void testMapper() throws IOException {
mapDriver.withInput(new LongWritable(), new Text("map的输入"));
mapDriver.withOutput(new Text("期望的key"), new Text("期望的value"));

//打印实际结果
List<Pair<Text , Text>> result = mapDriver.run();
for(Pair<Text , Text> kv : result){
System.out.println("mapper : " + kv.getFirst());
System.out.println("mapper : " + kv.getSecond());
}
//进行case测试，对比输入输出结果
mapDriver.runTest();
}

@Test
public void testReducer() throws IOException {
List<Text> values = new ArrayList<Text>();
values.add(new Text("输入"));
reduceDriver.withInput(new Text("输入"), values);
reduceDriver.withOutput(new Text("期望的输出"), new Text("期望的输出"));
reduceDriver.runTest();
}

@Test
public void testMapperReducer() throws IOException {
mapReduceDriver.withInput(new LongWritable(), new Text("输入"));
mapReduceDriver.withOutput(new Text("期望的输出"), new Text("期望的输出"));
//打印实际结果
List<Pair<Text, Text>> list =  mapReduceDriver.run();
System.out.println("mapreducedriver size:" + list.size());
for(Pair<Text , Text> lst : list){
System.out.println(lst.getFirst());
System.out.println(lst.getSecond());
}
//进行case测试，对比输入输出结果
mapReduceDriver.runTest();
}

@Test
public void testMapperCount() throws IOException {
mapDriver.withInput(new LongWritable(), new Text("输入"));
mapDriver.withOutput(new Text("期望的输出"), new Text("期望的输出"));
mapDriver.runTest();
//判断 map中的counter值是否与期望的相同
assertEquals("Expected 1 counter increment", 1, mapDriver.getCounters().findCounter("data", "suc").getValue());
}
}