您的位置:首页 > 运维架构

Hadoop模板(新API)

2016-03-22 17:01 465 查看
跟原来的API稍有不同,其实也差不多

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

import sun.launcher.resources.launcher;

public class NewAPIMyJob extends Configured implements Tool {

public static class MapClass extends Mapper<Text, Text, Text, Text>{
public void map(Text key,Text value,Context context)throws IOException,InterruptedException{
context.write(value, key);
}
}
public static class Reduce  extends Reducer<Text, Text, Text, Text>{

public void reduce(Text key,Iterable<Text> value,Context context)throws IOException,InterruptedException{

String reString="";
for(Text val:value){
if(reString.length()>0)reString+=",";
reString+=val.toString();
}
context.write(key, new Text(reString));
}
}
@Override
public int run(String[] arg0) throws Exception,InterruptedException {
// TODO Auto-generated method stub

Configuration configuration=getConf();
Job job=new Job(configuration,"MyJob");

job.setJarByClass(MyJob.class);

FileInputFormat.setInputPaths(job, new Path(arg0[0]));
FileOutputFormat.setOutputPath(job, new Path(arg0[1]));

job.setMapperClass(MapClass.class);
job.setReducerClass(Reduce.class);
job.setInputFormatClass(KeyValueTextInputFormat.class);
job.setOutputFormatClass(TextOutputFormat.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
System.exit(job.waitForCompletion(true)?0:1);

return 0;
}

public static void main(String[] args)throws Exception {
// TODO Auto-generated method stub

int res=ToolRunner.run(new Configuration(),new NewAPIMyJob(),args);
System.exit(res);

}

}
内容来自用户分享和网络整理,不保证内容的准确性,如有侵权内容,可联系管理员处理 点击这里给我发消息
标签: