MapReduce之SequenceFile转普通文件
2017-05-07 10:54
429 查看
package com.uplooking.bigdata.mr.writablez.sequecefile;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.SequenceFile.Reader;
import org.apache.hadoop.io.SequenceFile.Reader.Option;
import org.apache.hadoop.io.Text;
import java.io.*;
/**
* 将一个SequenceFile的文件转变成为一个普通的文本文件
*/
public class SequenceFileReadOps {
public static void main(String[] args) throws Exception {
if(args == null || args.length < 2) {
System.err.println("Parameter Errors ! Usage: <inputpath outputpath>");
System.exit(-1);
}
String inputpath = args[0];
// Path outputpath = new Path(args[1]);
String outputpath = args[1];
Configuration conf = new Configuration();
Option inputOps = Reader.file(new Path(inputpath));
Option[] opts = new Option[]{inputOps};
Reader reader = new SequenceFile.Reader(conf, opts);
Text key = new Text();
BufferedWriter bw = new BufferedWriter(new FileWriter(outputpath));
while(reader.next(key, NullWritable.get())) {//将数据写入到key里面了
String line = key.toString();
bw.write(line);
bw.newLine();
}
bw.close();
reader.close();
}
}
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.SequenceFile.Reader;
import org.apache.hadoop.io.SequenceFile.Reader.Option;
import org.apache.hadoop.io.Text;
import java.io.*;
/**
* 将一个SequenceFile的文件转变成为一个普通的文本文件
*/
public class SequenceFileReadOps {
public static void main(String[] args) throws Exception {
if(args == null || args.length < 2) {
System.err.println("Parameter Errors ! Usage: <inputpath outputpath>");
System.exit(-1);
}
String inputpath = args[0];
// Path outputpath = new Path(args[1]);
String outputpath = args[1];
Configuration conf = new Configuration();
Option inputOps = Reader.file(new Path(inputpath));
Option[] opts = new Option[]{inputOps};
Reader reader = new SequenceFile.Reader(conf, opts);
Text key = new Text();
BufferedWriter bw = new BufferedWriter(new FileWriter(outputpath));
while(reader.next(key, NullWritable.get())) {//将数据写入到key里面了
String line = key.toString();
bw.write(line);
bw.newLine();
}
bw.close();
reader.close();
}
}
相关文章推荐
- mapreduce作业接受序列化文件(SequenceFile)作为文件输入的WordCount程序
- 用CombineFileInputFormat处理小文件的mapreduce程序
- sequencefile处理小文件实例
- Hadoop MapReduce处理海量小文件:基于CombineFileInputFormat
- Hadoop MapReduce处理海量小文件:基于CombineFileInputFormat(每次往map中读入1行)
- Hadoop MapReduce处理海量小文件:基于CombineFileInputFormat(整个小文件读入到map中)
- SequenceFile 解决hadoop小文件问题
- mapreduce的文件拆分,FileInputFormat
- Hadoop Core 学习笔记(一) SequenceFile文件写入和读取Writable数据
- 例子2之解决ORA-03113: 通信通道的文件结尾(实质是Control filesequence number mismatch)
- 如何在mapreduce方法中获取当前使用文件(get file name)
- mapreduce的文件拆分,FileInputFormat
- Hadoop序列化文件SequenceFile
- Hadoop下将大量小文件生成一个sequenceFile文件
- MapReduce中的SequenceFile和MapFile
- SequenceFile,RCFile格式的表不能直接从本地文件导入数据
- MapReduce的SequenceFileOutputFormat使用
- Hadoop 写SequenceFile文件 源代码
- hadoop序列文件读与写 (SequenceWriteReadFile)
- Hadoop下将大量小文件生成一个sequenceFile文件