一个典型的MapRuduce实例------webcount（网站统计访客信息）

统计某一特定网站的某个时辰访客人数

所用版本：hadoop2.6.5

数据样式如下：

111.111.111.111 - - [16/Dec/2012:05:32:50 -0500] "GET / HTTP/1.1" 200 14791 "-" "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"
111.111.111.111 - - [16/Dec/2012:05:33:50 -0500] "GET / HTTP/1.1" 200 14791 "-" "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"
111.111.111.111 - - [16/Dec/2012:05:34:45 -0500] "GET / HTTP/1.1" 200 14791 "-" "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"
111.111.111.111 - - [16/Dec/2012:05:34:50 -0500] "GET / HTTP/1.1" 200 14791 "-" "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"
111.111.111.111 - - [16/Dec/2012:09:34:55 -0500] "GET / HTTP/1.1" 200 14791 "-" "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"
111.111.111.111 - - [16/Dec/2012:10:23:30 -0500] "GET / HTTP/1.1" 200 14791 "-" "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"
111.111.111.111 - - [16/Dec/2012:10:32:50 -0500] "GET / HTTP/1.1" 200 14791 "-" "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"

辅助类

1 package com.trendwise.software;
2
3 import java.text.SimpleDateFormat;
4 import java.util.Date;
5 import java.io.DataInput; import java.io.DataOutput;
6 import java.io.IOException;
7 import org.apache.hadoop.io.WritableComparable;
8
9 public class DateWritable implements WritableComparable<DateWritable>{
10     private final static SimpleDateFormat formatter = new SimpleDateFormat( "yyyy-MM-dd' T 'HH:mm:ss.SSS" );
11     private Date date;
12     public Date getDate() {
13         return date;
14     }
15     public void setDate( Date date ) {
16         this.date = date;
17     }
18
19     @Override
20     public void readFields(DataInput in) throws IOException {
21         date = new Date( in.readLong() );
22     }
23
24     @Override
25     public void write(DataOutput out) throws IOException {
26         out.writeLong( date.getTime() );
27     }
28
29     @Override
30     public int compareTo(DateWritable o) {
31         return date.compareTo( o.getDate() );
32     }
33
34     public String toString() {
35         return formatter.format( date);
36     }
37 }

mapper 映射特定年份中每月每天每个时辰的访客数

1 package com.trendwise.software;
2
3 import java.io.IOException;
4 import java.util.Calendar;
5 import org.apache.hadoop.io.IntWritable;
6 import org.apache.hadoop.io.LongWritable;
7 import org.apache.hadoop.io.Text;
8 import org.apache.hadoop.mapreduce.Mapper;
9
10 public class LogMapper extends Mapper<LongWritable, Text, DateWritable, IntWritable> {
11     public static DateWritable dates = new DateWritable();
12     public final static IntWritable two = new IntWritable(1);
13     public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
14         String text = value.toString();
15         // Get the date and time
16         int openBracket = text.indexOf( '[' );
17         int closeBracket = text.indexOf( ']' );
18         if( openBracket != -1 && closeBracket != -1 ) {
19             // Read the date
20             String dateString = text.substring( text.indexOf( '[' ) + 1, text. indexOf( ']' ) );
21             // Build a date object from a string of the form: 16/Dec/2012:05:32:50 -0500
22             int index = 0;
23             int nextIndex = dateString.indexOf( '/' );
24             int day = Integer.parseInt( dateString.substring(index, nextIndex) );
25
26             index = nextIndex; nextIndex = dateString.indexOf( '/', index+1 );
27             String month = dateString.substring( index+1, nextIndex );
28             index = nextIndex;
29             nextIndex = dateString.indexOf( ':', index );
30             int year = Integer.parseInt(dateString.substring(index + 1, nextIndex));
31             index = nextIndex; nextIndex = dateString.indexOf( ':', index+1 );
32             int hour = Integer.parseInt(dateString.substring(index + 1, nextIndex));
33             // Build a calendar object for this date
34             Calendar calendar = Calendar.getInstance();
35             calendar.set( Calendar.DATE, day );
36             calendar.set( Calendar.YEAR, year );
37             calendar.set( Calendar.HOUR, hour );
38             calendar.set( Calendar.MINUTE, 0 );
39             calendar.set( Calendar.SECOND, 0 );
40             calendar.set( Calendar.MILLISECOND, 0 );
41             if( month.equalsIgnoreCase( "dec" ) ) {
42                 calendar.set( Calendar.MONTH, Calendar.DECEMBER );
43             }
44             else if( month.equalsIgnoreCase( "nov" ) ) {
45                 calendar.set( Calendar.MONTH, Calendar.NOVEMBER );
46             }
47             else if( month.equalsIgnoreCase( "oct" ) ) {
48                 calendar.set( Calendar.MONTH, Calendar.OCTOBER );
49             }
50             else if( month.equalsIgnoreCase( "sep" ) ) {
51                 calendar.set( Calendar.MONTH, Calendar.SEPTEMBER );
52             }
53             else if( month.equalsIgnoreCase( "aug" ) ) {
54                 calendar.set( Calendar.MONTH, Calendar.AUGUST );
55             }
56             else if( month.equalsIgnoreCase( "jul" ) ) {
57                 calendar.set( Calendar.MONTH, Calendar.JULY );
58             }
59             else if( month.equalsIgnoreCase( "jun" ) ) {
60                 calendar.set( Calendar.MONTH, Calendar.JUNE );
61             }
62             else if( month.equalsIgnoreCase( "may" ) ) {
63                 calendar.set( Calendar.MONTH, Calendar.MAY );
64             }
65             else if( month.equalsIgnoreCase( "apr" ) ) {
66                 calendar.set( Calendar.MONTH, Calendar.APRIL );
67             }
68             else if( month.equalsIgnoreCase( "mar" ) ) {
69                 calendar.set( Calendar.MONTH, Calendar.MARCH );
70             }
71             else if( month.equalsIgnoreCase( "feb" ) ) {
72                 calendar.set( Calendar.MONTH, Calendar.FEBRUARY );
73             }
74             else if( month.equalsIgnoreCase( "jan" ) ) {
75                 calendar.set( Calendar.MONTH, Calendar.JANUARY );
76             }
77
78             dates.setDate( calendar.getTime() );
79             context.write(dates, two);
80
81         }
82     }
83 }

reducer 汇总一个时辰内访客人数

1 package com.trendwise.software;
2
3 import java.io.IOException;
4 import org.apache.hadoop.io.IntWritable;
5 import org.apache.hadoop.mapreduce.Reducer;
6
7 public class  LogReducer extends Reducer<DateWritable, IntWritable, DateWritable, IntWritable> {
8     @Override
9     public void reduce( DateWritable key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
10
11         int countn = 0;
12         for(IntWritable v :values){
13             countn += v.get();
14         }
15         context.write(key, new IntWritable( countn) );
16     }
17 }

driver 配置信息，程序入口

1 package com.trendwise.software;
2
3 import java.io.IOException;
4 import org.apache.hadoop.conf.Configuration;
5 import org.apache.hadoop.fs.Path;
6 import org.apache.hadoop.io.IntWritable;
7 import org.apache.hadoop.mapreduce.Job;
8 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
9 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
10
11 public class Driver {
12
13     public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
14
15         String in = args[0];
16         String out = args[1];
17         int unitmb =Integer.valueOf(args[2]);
18         int nreducer = Integer.valueOf(args[3]);
19
20         Configuration conf = new Configuration();
21         conf.set("mapreduce.input.fileinputformat.split.maxsize", String.valueOf(unitmb * 1024 * 1024));
22         conf.set("mapred.min.split.size", String.valueOf(unitmb * 1024 * 1024));
23         conf.set("mapreduce.input.fileinputformat.split.minsize.per.node", String.valueOf(unitmb * 1024 * 1024));
24         conf.set("mapreduce.input.fileinputformat.split.minsize.per.rack", String.valueOf(unitmb * 1024 * 1024));
25
26         Job job = new Job(conf);
27         FileInputFormat.addInputPath(job, new Path(in));
28         FileOutputFormat.setOutputPath(job, new Path(out));
29         job.setMapperClass(LogMapper.class);
30         job.setReducerClass(LogReducer.class);
31         job.setCombinerClass(LogReducer.class);
32         job.setNumReduceTasks(nreducer);
33         job.setMapOutputKeyClass(DateWritable.class);
34         job.setMapOutputValueClass(IntWritable.class);
35         job.setOutputKeyClass(DateWritable.class);
36         job.setOutputValueClass(IntWritable.class);
37         job.setJarByClass(Driver.class);
38         job.waitForCompletion(true);
39
40     }
41 }

command



result