日志文件的处理和分析

package com.tudou.thread;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.lang.Thread.State;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class ReadProcessedFile {

private static BufferedReader bReader = null;
private static String content = null;
private static String splitInfo[] = null;
private static final int COUNT = 1;

//	private static  int test_int = 0;

/*
* 读取文件，取出所有的INFO日志存于另一个文件中
*/
public static void readFile(String log,String log_info) throws IOException {

File inputfile = new File(log);
bReader = new BufferedReader(new FileReader(inputfile));
while((content = bReader.readLine()) != null) {
System.out.println(content);
if(content.startsWith("2014-12-09")){
splitInfo = content.split(" ");
if(splitInfo[3].equals("INFO")) {
writeFile(content,log_info);
}
}
}
bReader.close();
}
/*
* 内容写入文件
*/
public static void writeFile(String content,String log_info) throws IOException {
File outputFile = new File(log_info);
if(!outputFile.exists()) {
outputFile.createNewFile();
}

FileWriter fWriter = new FileWriter(outputFile.getAbsoluteFile(),true);
fWriter.write(content+"\n");
fWriter.close();
}

/*
* map对象写入文件
*/
public static void writeObjectToFile(Map<String,Integer> map,String final_result) throws IOException{
File file = new File(final_result);
if(!file.exists()){
file.createNewFile();
}
FileWriter fWriter = new FileWriter(file.getAbsoluteFile(),true);
StringBuilder param = new StringBuilder();
for(String key : map.keySet()) {
param.append(key+"="+map.get(key)+",");
}
fWriter.write(param.toString()+"\n\n");
fWriter.close();
//		System.out.println("成功写入文件");
}

/*
* 将INFO日志的content部分存到另一个文件中；
*/
public static void readProcessedFile(String log_info,String info_content) throws IOException {
File file = new File(log_info);
bReader = new BufferedReader(new FileReader(file));
while((content = bReader.readLine()) != null){
System.out.println(content);
splitInfo = content.split(" - ");
if(splitInfo.length > 1){
writeFile(splitInfo[1], info_content);
}
}
bReader.close();
}

/*
* 统计各个s3_app类型的访问量
*/
public static void readLogContent(String info_content,String final_result) throws IOException {
File file = new File(info_content);
bReader = new BufferedReader(new FileReader(file));
String s3_app = null;
Map<String, Integer> map = null;
while((content = bReader.readLine()) != null){
if(content.trim().startsWith("s3__app=")){
splitInfo = content.split(" ");
s3_app = splitInfo[0].substring(9,splitInfo[0].length()-1);
if(map == null) {
map = new HashMap<String, Integer>();
map.put(s3_app, COUNT) ;
}else {
int state = 0;
for(String key : map.keySet()){
state ++;
if(s3_app.equals(key)){
map.put(s3_app, map.get(s3_app)+1);
break;
}
if(state == map.size()){
map.put(s3_app, COUNT);
}
}
}
}
}
bReader.close();
writeObjectToFile(map,final_result);
System.out.println("Each s3_app Model的访问量->" + map);
}

/*
* s3_app访问 Each app Model的访问量
*/

public static void getVisitedInfoByLogContent(String info_content,String final_result) throws IOException{
File file = new File(info_content);
bReader = new BufferedReader(new FileReader(file));
Map<String, Integer> map = null;
String visitInfo = null;
//		String s3_app = null;
while((content = bReader.readLine()) != null){
if(content.trim().startsWith("s3__app=")){
splitInfo = content.split(" ");
visitInfo = splitInfo[0].substring(9,splitInfo[0].length()-1).concat("-"+splitInfo[5].substring(5,splitInfo[5].length()-1));
if(map == null) {
map = new HashMap<String ,Integer>();
map.put(visitInfo, COUNT);
}else {
int state = 0;
for(String key : map.keySet()){
state ++;
if(visitInfo.equals(key)){
map.put(visitInfo, map.get(visitInfo)+1);
break;
}
if(state == map.size()){
map.put(visitInfo,COUNT);
}
}
}
}
}
bReader.close();
System.out.println("s3_app each app Model的访问量->"+map);
writeObjectToFile(map,final_result);
}

/*
* each app param 的 访问量
*/

public static void getEachParamInfoByLogContent(String info_content,String final_result) throws IOException {

File file = new File(info_content);
bReader = new BufferedReader(new FileReader(file));
//		List<String> paramList = new ArrayList<String>() ;
String param[] = null;
//		String eachParam[] = null;
String paramArray[] = null;
String paramInfo[] = null;
//		String splitParamInfo[] = null;
Map<String,String> paramValue = new HashMap<String,String>();
Map<String,Integer> statisticsParam = new HashMap<String,Integer>();
StringBuilder totalParam = new StringBuilder();

while((content = bReader.readLine()) != null){
//			test_int ++;
if(content.trim().startsWith("s3__app=")){
splitInfo = content.split(" FakeSolrParam ");
param = splitInfo[1].split(",");
int state = -1;
int conLocation = 0;
int memLocation = 0;
for(String eachParam : param) {
//					System.out.println(eachParam);
state ++;
if(eachParam.trim().startsWith("[app")){
totalParam.append(eachParam.trim()+",");
}
if(eachParam.trim().startsWith("sortAsc")){
totalParam.append(eachParam.trim()+",");
continue;
}
if(eachParam.trim().startsWith("sortDesc")){
totalParam.append(eachParam.trim()+",");
continue;
}
if(eachParam.trim().startsWith("conditionFields")){
//						totalParam.append(eachParam+",");
conLocation = state;
//						System.out.println("conLocation="+conLocation);
continue;

}
if(eachParam.trim().startsWith("memcacheTime")){
memLocation = state;
//						System.out.println("memLocation="+memLocation);
}
}

for(int i=conLocation;i<memLocation;i++){
totalParam.append(param[i].trim()+"-");
}
paramArray = totalParam.toString().split(",");
for(String param1:paramArray) {
paramInfo = param1.split("=");
paramValue.put(paramInfo[0], paramInfo[1]);
}
//				System.out.println(test_int);
//				System.out.println(paramValue);
if(paramValue.get("sortAsc") != null && !paramValue.get("sortAsc").trim().equals("[]")){
String key = paramValue.get("[app")+"-"+"sortAsc"+"-"+paramValue.get("sortAsc").substring(1, paramValue.get("sortAsc").length()-1);
if(statisticsParam.size()==0){
statisticsParam.put(key, COUNT);
}else {
int state1 = 0;
for(String statisticsKey:statisticsParam.keySet()) {
state1 ++;
if(key.equals(statisticsKey)){
statisticsParam.put(statisticsKey, statisticsParam.get(statisticsKey)+1);
break;
}
if(statisticsParam.size() == state1){
statisticsParam.put(key, COUNT);
}
}
}
}

if(paramValue.get("sortDesc") != null && !paramValue.get("sortDesc").trim().equals("[]")){
String key = paramValue.get("[app")+"-"+"sortDesc"+"-"+paramValue.get("sortDesc").substring(1,paramValue.get("sortDesc").length()-1);
if(statisticsParam.size()==0){
statisticsParam.put(key, COUNT);
}else {
int state2 = 0;
for(String statisticsKey:statisticsParam.keySet()) {
state2 ++;
if(key.equals(statisticsKey)){
statisticsParam.put(statisticsKey, statisticsParam.get(statisticsKey)+1);
break;
}

if(statisticsParam.size() == state2){
statisticsParam.put(key, COUNT);
}
}
}
}

if(paramValue.get("conditionFields") != null && !paramValue.get("conditionFields").equals("[]")){
String conditionParam[] = paramValue.get("conditionFields").split("-");
if(conditionParam.length != 1){
int flag = 0;
for(String conditionValue : conditionParam){
flag ++;
String key = null;
if(flag == 1) {
key = paramValue.get("[app")+"-"+"fq"+"-"+conditionValue.substring(1,conditionValue.length());
}
else if(flag == conditionParam.length){
key = paramValue.get("[app")+"-"+"fq"+"-"+conditionValue.substring(0,conditionValue.length()-1);
}else{
key = paramValue.get("[app")+"-"+"fq"+"-"+conditionValue.substring(0,conditionValue.length());
}

if(!key.equalsIgnoreCase(paramValue.get("[app")+"-fq-null") && !key.equalsIgnoreCase(paramValue.get("[app")+"-fq-")){
if(statisticsParam.size() == 0){
statisticsParam.put(key, COUNT);
}else {
int state3 = 0;
for(String statisticsKey:statisticsParam.keySet()) {
state3 ++;
if(key.equals(statisticsKey)){
statisticsParam.put(statisticsKey, statisticsParam.get(statisticsKey)+1);
break;
}

if(statisticsParam.size() == state3){
statisticsParam.put(key, COUNT);
}
}
}
}
}
}else{
String key = paramValue.get("[app")+"-"+"fq"+"-"+paramValue.get("conditionFields").substring(1,paramValue.get("conditionFields").length()-2);
if(!key.equalsIgnoreCase(paramValue.get("[app")+"-fq-null") && !key.equalsIgnoreCase(paramValue.get("[app")+"-fq-")){
if(statisticsParam.size()==0){
statisticsParam.put(key, COUNT);
}else {
int state4 = 0;
for(String statisticsKey:statisticsParam.keySet()) {
state4 ++;
if(key.equals(statisticsKey)){
statisticsParam.put(statisticsKey, statisticsParam.get(statisticsKey)+1);
break;
}

if(statisticsParam.size() == state4){
statisticsParam.put(key, COUNT);
}
}
}
}
}
}
paramValue.clear();
totalParam.delete(0, totalParam.length());
}
}
bReader.close();
System.out.println("each app param 的 访问量->"+statisticsParam);
writeObjectToFile(statisticsParam,final_result);
}

public static void main(String args[]) throws IOException {
readFile("D:\\s3_interface.log.2014-12-09", "D:\\log_Info.txt");
readProcessedFile("D:\\log_Info.txt","D:\\info_content.txt");
readLogContent("D:\\info_content.txt","D:\\final_result.txt");
getVisitedInfoByLogContent("D:\\info_content.txt","D:\\final_result.txt");
getEachParamInfoByLogContent("D:\\info_content.txt","D:\\final_result.txt");

}
}