日志分析   取出访问最多的IP，URL，以及五分钟内的访问流量

#!/bin/sh

name=`cat /etc/salt/minion | grep "^id" | awk '{print $2}'`
ipaddr=`/sbin/ifconfig | grep "inet addr" | egrep -v "10\.|127\." | awk  -F'[: ]+' '{print $4}'`
#LogPath='/usr/local/tengine/logs/'

##日志的时间格式。
function Time(){
FileTime=`head -1 $Logfile |awk -F"[[ / :]" '{print "["$5"/"$6"/"$7}'`
start_time=${FileTime}:`date -d "6 mins ago" +%H:%M`
#now_time=${FileTime}:`date  +%H:%M`
}

##取得指定最新时间内的日志，并进行分析
function IpUrlFlow(){
Time
path="/tmp/backup/log"
[ ! -d $path/flow/ ] && mkdir -p $path/flow
file="$path/${Logfile}.time.log"
FileIp="$path/${Logfile}.IP.log"
FileUri="$path/${Logfile}.uri.sort.log"
FileIpStat="$path/${Logfile}.flow.sort.log"
FileStat="$path/flow/${Logfile}.flow.log"
#取出直接时间内的日志
#awk -v start_time=$start_time -v now_time=$now_time  '{if ($4>start_time &&  $4<now_time) print $0 }' ${Logfile} > $file
awk -v start_time=$start_time -v now_time=$now_time  '{if ($4>start_time) print $0 }' ${Logfile} > $file
if [  -s $file ]
then
#对取出的日志进行统计，取出访问次数最多的ip以及次数
awk '{S[$1]++}END{for(n in S) print S
"      "n }'  $file  |sort -rn |head -20  > ${FileIp}
#awk '{S[$1]++}END{for(n in S) print S
"      "n }'  $file  |sort -rn |head -20  > ${FileIpStat}
#对取出的日志进行统计，取出访问次数最多的URI以及次数
awk '{S[$7]++}END{for(n in S) print S
"      "n }'  $file  |sort -rn |head -20  > ${FileUri}
#awk '{S[$7]++}END{for(n in S) print S
"      "n }'  $file  |sort -rn |head -20  >> ${FileIpStat}
#对取出的日志进行统计，取出访问流量最多的ip以及流量大小，单位M
awk '{S[$1]=+$10}END{for(n in S) print S
"      "n }' $file |sort -rn|head -20 |awk '{a=$1/1024/1014}{print a " M          "$2}' >${FileIpStat}
#awk '{S[$1]=+$10}END{for(n in S) print S
"      "n }' $file |sort -rn|head -20 |awk '{a=$1/1024/1014}{print a " M          "$2}' >>${FileIpStat}

#整合到一个文件！
echo -e "IP_start" > ${FileStat}
cat ${FileIp} >> ${FileStat}
echo -e "IP_end\n"  >> ${FileStat}
echo -e "URL_start"  >> ${FileStat}
cat ${FileUri} >> ${FileStat}
echo -e "URL_end\n"  >> ${FileStat}
echo -e "Flow_star"  >> ${FileStat}
cat ${FileIpStat} >> ${FileStat}
echo -e "Flow_end\n"  >> ${FileStat}
fi
}

#Ip
function main(){
cd ${LogPath}
for Logfile in `ls acce*`
do
if [ ! -s ${Logfile} ]
then
echo ${Logfile}
else
Ip
fi

done

}
main

###############################################################################
###############################################################################
###############################################################################
#!/bin/sh

name=`cat /etc/salt/minion | grep "^id" | awk '{print $2}'`
ipaddr=`/sbin/ifconfig | grep "inet addr" | egrep -v "10\.|127\." | awk  -F'[: ]+' '{print $4}'`
LogPath='/usr/local/tengine/logs/'
#filelog='access_www.log'
#Logfile="${LogPath}${filelog}"

function Time(){
FileTime=`head -1 $Logfile |awk -F"[[ / :]" '{print "["$5"/"$6"/"$7}'`
start_time=${FileTime}:`date -d "6 mins ago" +%H:%M`
now_time=${FileTime}:`date  +%H:%M`

#echo $FileTime $start_time $now_time
}

function Ip(){
Time
path="/script/shell/gaogd/LogAnalyze/log"
[ ! -d $path/flow ] && mkdir -p $path/flow
file="$path/${Logfile}.time.log"
#FileIp="$path/${Logfile}.IP.log"
#FileUri="$path/${Logfile}.uri.sort.log"
#FileIpStat="$path/${Logfile}.stat.sort.log"
FileStat="$path/flow/${Logfile}.Statistics.log"
awk -v start_time=$start_time -v now_time=$now_time  '{if ($4>start_time &&  $4<now_time) print $0 }' ${Logfile} > $file
if [  -s $file ]
then
echo -e "IP_start" > ${FileStat}
awk '{S[$1]++}END{for(n in S) print S
"      "n }'  $file  |sort -rn |head -20 >> ${FileStat}
echo -e "IP_end\n"  >> ${FileStat}
echo -e "URL_start"  >> ${FileStat}
awk '{S[$7]++}END{for(n in S) print S
"      "n }'  $file  |sort -rn |head -20  >> ${FileStat}
echo -e "URL_end\n"  >> ${FileStat}
echo -e "Flow_star"  >> ${FileStat}
awk '{S[$1]=+$10}END{for(n in S) print S
"      "n }' $file |sort -rn|head -20 |awk '{a=$1/1024/1014}{print a " M          "$2}' >>${FileStat}
echo -e "Flow_end\n"  >> ${FileStat}
else
if [  -s ${FileStat} ]
then
mv ${FileStat} /tmp
fi
fi
}

#Ip
function main(){
cd ${LogPath}
for Logfile in `ls acce*`
do
if [ ! -s ${Logfile} ]
then
echo ${Logfile}
else
Ip
fi

done

}
main