使用python logging处理多机多进程写同一个日志文件

MemoryHandler的性能问题:

如果target是StreamHandler的子类

上是有严重的IO性能问题

是一个for调用handler，

handler中的处理侧是 io.write(), io.flush()

立马的flush到硬盘中，并有多次flush，io性能很差



logging模块本身是支持多线程写同一个文件的。但对多进程写同一个文件并没有现在成的代码支持。
如tornado的多进程模式与django的fastcgi （flup）多进程模式的场境，写日志都可以应用以下代码:

CS结构

server代码如下, 应该是09 年的项目代码:

#coding:utf8
#author:TooNTonG 2011-11-07

from SocketServer import ThreadingTCPServer, StreamRequestHandler
import logging.config
import logging.handlers as lhandlers
import os
import struct
import cPickle

LOG_BIND_PORT = 20001

class LogRequestHandler(StreamRequestHandler):
def handle(self):
while 1:
chunk = self.connection.recv(4)
if len(chunk) < 4:
break
slen = struct.unpack(">L", chunk)[0]
chunk = self.connection.recv(slen)
while len(chunk) < slen:
chunk = chunk + self.connection.recv(slen - len(chunk))
obj = self.unPickle(chunk)
# 使用SocketHandler发送过来的数据包，要使用解包成为LogRecord
# 看SocketHandler文档
record = logging.makeLogRecord(obj)
self.handleLogRecord(record)

def unPickle(self, data):
return cPickle.loads(data)

def handleLogRecord(self, record):
logger = logging.getLogger(record.name)
logger.handle(record)

def startLogSvr(bindAddress, requestHandler):
svr = ThreadingTCPServer(bindAddress, requestHandler)
svr.serve_forever()

def addHandler(name, handler):
logger = logging.getLogger(name)
logger.addHandler(handler)

fmt = logging.Formatter('%(asctime)s - %(levelname)s - %(name)s - %(message)s')
handler.setFormatter(fmt)

logger.setLevel(logging.NOTSET)

def memoryWapper(handler, capacity):
hdlr = lhandlers.MemoryHandler(capacity, target = handler)
hdlr.setFormatter(handler.formatter)
return hdlr

def main():
path, dirname = os.path, os.path.dirname
pth = dirname((path.realpath(__file__)))
filename = path.join(dirname(pth), 'log', 'logging.log')
#    logging.config.fileConfig(pth + r'/logging.conf')

# 最终写到文件中
hdlr = lhandlers.RotatingFileHandler(filename,
maxBytes = 1024,
backupCount = 3)

# 还可以一个memoryhandler，达到一定数据或是有ERROR级别再flush到硬盘
hdlr = memoryWapper(hdlr, 1024)

addHandler('core', hdlr)

print 'OK: logerserver running...'
startLogSvr(('0.0.0.0', LOG_BIND_PORT), LogRequestHandler)

if __name__ == "__main__":
main()

再帖上客户端代码：

#coding:utf8
#author: TooNTonG 2012-11-07

import logging
import logging.handlers as handlers

APP_NAME = 'app1'
LOG_SVR_HOST = '127.0.0.1'
LOG_SVR_PORT = 20001

# 此logger name必需与服务端中有相应的logger处理handler
# 如果服务端logging.getLogger()返回空，会使用root处理
LOGGER_NAME = 'core'

def getSocketLogger(name, level, host, port, memoryCapacity):
target = handlers.SocketHandler(host, port)
if memoryCapacity > 0:

hdlr = handlers.MemoryHandler(memoryCapacity,
logging.ERROR, # 此参数是指遇到此级别时，马上flush
target)
else:
hdlr = target

hdlr.setLevel(level)
logger = logging.getLogger(name)
logger.addHandler(hdlr)
logger.setLevel(level)
return logger

def main():
logger = getSocketLogger(LOGGER_NAME,
logging.DEBUG, # 如果使用NOTSET，相当warning
host = LOG_SVR_HOST,
port = LOG_SVR_PORT,
memoryCapacity = 1024)

for i in range(10):
logger.info('run %s main' % APP_NAME)
logger.debug('thisis the debug log by %s' % APP_NAME)
logger.warning('thisis the warning log by %s' % APP_NAME)
logger.error('thisis the error log by %s' % APP_NAME)
logger.critical('thisis the critical log by %s' % APP_NAME)

print 'end main'

if '__main__' == __name__:
main()

如果不设置带名字的logger，就是统一处理了。设置带名字的好处是可以N个不同功能的进程、不在机器上的，服务使用一个logger-server就可以了。

logging.handlers中有很多handler，可以自行进行组装：