Java处理重复，重复的元素只保留一个。

一、第一个类。读取文字存储在LinkedList中，通过它的contains方法判断是否重复，重复不处理，不重复则加入。

package com.aawant.nlp.filehandle;

import java.io.BufferedReader;

import java.io.File;

import java.io.FileInputStream;

import java.io.InputStreamReader;

import java.util.LinkedList;

public class DeleteRepeat {

/**

* 将文件中的文字存储在LinkedList中，

* @param path

* @return

* @throws Exception

*/

public static LinkedList<String> deleterepeat(String path) throws Exception{

LinkedList<String> linkedlist = new LinkedList<String>();

File file = new File(path);

FileInputStream fi = new FileInputStream(file);

InputStreamReader ir = new InputStreamReader(fi);

@SuppressWarnings("resource")

BufferedReader br = new BufferedReader(ir);

String temp = "";

while((temp=br.readLine())!=null){

temp = temp.trim();

if(linkedlist.contains(temp)){

continue;

}

linkedlist.add(temp);

}

return linkedlist;

}

public static void main(String args[]) throws Exception{

;

String str = FileHandle.list2string(deleterepeat("F:/position.txt"));

FileHandle.string2File(str,"F:/position2.txt","utf-8");

}

}

二、第二个类。一些文件的基本处理方法。

package com.aawant.nlp.filehandle;

import java.io.BufferedWriter;

import java.io.File;

import java.io.FileOutputStream;

import java.io.OutputStreamWriter;

import java.util.Iterator;

import java.util.LinkedList;

public class FileHandle {

/**

* 将字符串写入 文件

* @param str 字符串

* @param outputPath 写入的路径

* @param code 写入的编码格式

* @throws Exception

*/

public static void string2File(String str, String outputPath, String code) throws Exception{

OutputStreamWriter osw = null;

FileOutputStream fos = null;

BufferedWriter bw = null;

File file = new File(outputPath);

fos = new FileOutputStream(file);

osw = new OutputStreamWriter(fos,code);

bw = new BufferedWriter(osw);

bw.write(str);

bw.close();

fos.close();

}

/**

* 将list中的数据以字符串形式输出

* @param linkedlist

* @return String

*/

public static String list2string(LinkedList<?> linkedlist){

Iterator<?> iter = linkedlist.iterator();

StringBuilder sb = new StringBuilder();

while(iter.hasNext()){

sb.append((String) iter.next());

sb.append("\n");

}

return sb.toString();

}

}