java.lang.NoSuchMethodException: MapReduce.WordCount$MyMapper.<init>()
2016-07-02 09:51
501 查看
今天在YARN平台上测试了一个程序–wordcount:
运行结果:
找了很长时间:终于发现MyMapper类与MyReducer类前面忘加了static,加上之后,程序运行正确!
package MapReduce; import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; import org.apache.hadoop.mapreduce.lib.partition.HashPartitioner; public class WordCount { public static String path1 = "hdfs://hadoop11:9000/dirdata";//读取HDFS中的测试集 public static String path2 = "hdfs://hadoop11:9000/worddir"; public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); conf.set("fs.defaultFS", "hdfs://hadoop11:9000"); FileSystem fileSystem = FileSystem.get(conf);//获取HDFS的一个客户端实例 if(fileSystem.exists(new Path(path2))) { fileSystem.delete(new Path(path2), true); } Job job = Job.getInstance(conf); job.setJarByClass(WordCount.class); FileInputFormat.setInputPaths(job, new Path(path1)); job.setInputFormatClass(TextInputFormat.class); job.setMapperClass(MyMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(LongWritable.class); job.setNumReduceTasks(1); job.setPartitionerClass(HashPartitioner.class); job.setReducerClass(MyReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(LongWritable.class); job.setOutputFormatClass(TextOutputFormat.class); FileOutputFormat.setOutputPath(job, new Path(path2)); job.waitForCompletion(true); } public class MyMapper extends Mapper<LongWritable, Text, Text, LongWritable> { protected void map(LongWritable k1, Text v1,Context context)throws IOException, InterruptedException { String[] splited = v1.toString().split("\t"); for (String string : splited) { context.write(new Text(string),new LongWritable(1L)); } } } public class MyReducer extends Reducer<Text, LongWritable, Text, LongWritable> { protected void reduce(Text k2, Iterable<LongWritable> v2s,Context context)throws IOException, InterruptedException { long sum = 0L; for (LongWritable v2 : v2s) { sum += v2.get(); } context.write(k2,new LongWritable(sum)); } } }
运行结果:
Error: java.lang.RuntimeException: java.lang.NoSuchMethodException: MapReduce.WordCount$MyMapper.<init>() at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:131) at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:722) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:340) at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:167) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1556) at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:162) Caused by: java.lang.NoSuchMethodException: MapReduce.WordCount$MyMapper.<init>() at java.lang.Class.getConstructor0(Class.java:2800) at java.lang.Class.getDeclaredConstructor(Class.java:2043) at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:125) ... 7 more
找了很长时间:终于发现MyMapper类与MyReducer类前面忘加了static,加上之后,程序运行正确!
package MapReduce; import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; import org.apache.hadoop.mapreduce.lib.partition.HashPartitioner; public class WordCount { public static String path1 = "hdfs://hadoop11:9000/dirdata";//读取HDFS中的测试集 public static String path2 = "hdfs://hadoop11:9000/worddir"; public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); conf.set("fs.defaultFS", "hdfs://hadoop11:9000"); FileSystem fileSystem = FileSystem.get(conf);//获取HDFS的一个客户端实例 if(fileSystem.exists(new Path(path2))) { fileSystem.delete(new Path(path2), true); } Job job = Job.getInstance(conf); job.setJarByClass(WordCount.class); FileInputFormat.setInputPaths(job, new Path(path1)); job.setInputFormatClass(TextInputFormat.class); job.setMapperClass(MyMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(LongWritable.class); job.setNumReduceTasks(1); job.setPartitionerClass(HashPartitioner.class); job.setReducerClass(MyReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(LongWritable.class); job.setOutputFormatClass(TextOutputFormat.class); FileOutputFormat.setOutputPath(job, new Path(path2)); job.waitForCompletion(true); } public static class MyMapper extends Mapper<LongWritable, Text, Text, LongWritable> { protected void map(LongWritable k1, Text v1,Context context)throws IOException, InterruptedException { String[] splited = v1.toString().split("\t"); for (String string : splited) { context.write(new Text(string),new LongWritable(1L)); } } } public static class MyReducer extends Reducer<Text, LongWritable, Text, LongWritable> { protected void reduce(Text k2, Iterable<LongWritable> v2s,Context context)throws IOException, InterruptedException { long sum = 0L; for (LongWritable v2 : v2s) { sum += v2.get(); } context.write(k2,new LongWritable(sum)); } } }
相关文章推荐
- 李洪强iOS开发之OC[007] - 李洪强iOS开发之类的声明和实现
- Android官方数据绑定框架DataBinding
- android xml文件不能显示视图 解决办法
- Unity中的C#学习(一)
- 策略模式
- 2016:iOS开发趋势思考--写的有趣~
- android 自定义View
- Android App 用户注册与登录实战(结合服务端)
- Swift - 使用NSURL进行数据的提交和获取(POST与GET)
- android studio获取sha1签名
- android fragment不执行onResume,onPause
- Unity如何发布一个Android 的开发应用
- Android自动连接指定的wifi,免密码或指定密码
- Android 项目中遇到的坑,特此记录
- GCD学习 - swift GCD使用指南
- android 中使用jwt token(json web token)--java
- IOS生成同时支持armv7,armv7s,i386,x86_64,arm64的静态库.a文件
- 高仿ios钉钉菜单展示效果
- 公司app出现异常版本与使用记录,怀疑app被换壳,下面总结换课过程,保护过程下节更新
- Android中WebView加载本地Html,与JavaScript与Android方法相互传值(续)