Hadoop模板(新API)
2016-03-22 17:01
465 查看
跟原来的API稍有不同,其实也差不多
import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; import sun.launcher.resources.launcher; public class NewAPIMyJob extends Configured implements Tool { public static class MapClass extends Mapper<Text, Text, Text, Text>{ public void map(Text key,Text value,Context context)throws IOException,InterruptedException{ context.write(value, key); } } public static class Reduce extends Reducer<Text, Text, Text, Text>{ public void reduce(Text key,Iterable<Text> value,Context context)throws IOException,InterruptedException{ String reString=""; for(Text val:value){ if(reString.length()>0)reString+=","; reString+=val.toString(); } context.write(key, new Text(reString)); } } @Override public int run(String[] arg0) throws Exception,InterruptedException { // TODO Auto-generated method stub Configuration configuration=getConf(); Job job=new Job(configuration,"MyJob"); job.setJarByClass(MyJob.class); FileInputFormat.setInputPaths(job, new Path(arg0[0])); FileOutputFormat.setOutputPath(job, new Path(arg0[1])); job.setMapperClass(MapClass.class); job.setReducerClass(Reduce.class); job.setInputFormatClass(KeyValueTextInputFormat.class); job.setOutputFormatClass(TextOutputFormat.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); System.exit(job.waitForCompletion(true)?0:1); return 0; } public static void main(String[] args)throws Exception { // TODO Auto-generated method stub int res=ToolRunner.run(new Configuration(),new NewAPIMyJob(),args); System.exit(res); } }
相关文章推荐
- 基于tiny4412的Linux内核移植(支持device tree)(二)
- Hadoop程序基础模板
- 基于tiny4412的Linux内核移植(支持device tree)(一)
- linux用sendmail发邮件
- OPENCV之GFTT特征点检测
- 编译 Linux 3.5 内核烧写 Android 到tiny4412
- 《软件架构设计》学习笔记--2--软件架构概念
- Hadoop简介
- Linux下命令符操作(一)
- 《Linux内核设计与实现》学习总结 Chap5
- linux --> 系统信息命令
- linux内核模块时内核树的作用
- centos 6.5 搭建svn
- Linux中变量$#,$@,$0,$1,$2,$*,$$,$?的含义
- 使用systemd配置SVN服务器自动启动
- Shell-记录-2
- Linux将多网口虚拟为一个网卡
- 【3】Nginx服务器基本配置简述
- org.apache.jasper.JasperException
- Linux的启动流程