hadoop编程之mapreduce,计算总数和平均数
2017-05-06 22:41
399 查看
import java.io.IOException;
import java.util.StringTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
public class Test {
public static class TokenizerMapper
extends Mapper<Object, Text, Text, Text>{
private Text record = new Text();
private Text time = new Text();
public void map(Object key, Text value, Context context
) throws IOException, InterruptedException {
StringTokenizer itr = new StringTokenizer(value.toString(), "\n");
while (itr.hasMoreTokens()) {
String[] strs = itr.nextToken().split("\\s+");
if(strs.length == 3) {
record.set(strs[0] + " " + strs[1]);
time.set(strs[2]);
context.write(record, time);
}
}
}
}
public static class AverageReducer
extends Reducer<Text, Text, Text, Text> {
private Text result = new Text();
private Text record = new Text();
public void reduce(Text key, Iterable<Text> values,
Context context
) throws IOException, InterruptedException {
int count = 0;
float aveScore = 0;
for (Text val : values) {
aveScore += Float.parseFloat(val.toString());
count++;
}
aveScore = aveScore / count;
record.set(key + " " + String.valueOf(count));
result.set(String .format("%.3f",aveScore));
context.write(record, result);
System.out.println(record.toString() + " " + String .format("%.3f",aveScore));
}
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
if(otherArgs.length < 2) {
System.err.println("Usage: wordcount <in> [<in>...] <out>");
System.exit(2);
}
Job job = new Job(conf, new Test().getClass().getName());
job.setJarByClass(Test.class);
job.setMapperClass(TokenizerMapper.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
job.setReducerClass(AverageReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
for (int i = 0; i < otherArgs.length - 1; ++i) {
FileInputFormat.addInputPath(job, new Path(otherArgs[i]));
}
FileOutputFormat.setOutputPath(job,
new Path(otherArgs[otherArgs.length - 1]));
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}
相关文章推荐
- Hadoop学习笔记—4.初识MapReduce 一、神马是高大上的MapReduce MapReduce是Google的一项重要技术,它首先是一个编程模型,用以进行大数据量的计算。对于大数据
- Hadoop并行计算原理与分布式并发编程
- Hadoop系列之三:函数式编程语言和MapReduce
- Hadoop集群_WordCount运行详解--MapReduce编程模型
- Hadoop2.2.0 第一步完成MapReduce wordcount计算文本数量
- MapReduce编程实例(一)-求平均数
- Hadoop是Apache提出的一个软件框架(即:开放源码并行运算编程工具和分布式文件系统,与MapReduce和Google档案系统的概念类似)
- Hadoop 稀疏矩阵乘法的MapReduce计算
- 用 Hadoop 的 MapReduce 编程实现 K-Means 算法
- hadoop 中的mapreduce编程模板
- Hadoop实战-中高级部分 之 Hadoop MapReduce高级编程
- Hadoop实战-中高级部分 之 Hadoop MapReduce高级编程
- [Hadoop编程实践]一个实用、清晰的MapReduce程序
- hadoop初学之MapReduce编程模型学习
- 云计算(二十五)- Hadoop MapReduce Next Generation - Writing YARN Applications
- 利用hadoop mapreduce进行 并行计算
- Hadoop系列之三:函数式编程语言和MapReduce
- 文件数据云计算学习笔记---Hadoop HDFS和MapReduce 架构浅析
- hadoop中使用MapReduce编程实例(转)
- hadoop中使用MapReduce编程实例