pgloader

使用pgloader加载数据到postgresql中

替换conf文件的日期

sed -i “s/[0-9]{4}-[0-9]{2}-[0-9]{2}/${YESTERDAY}/” /export/loganaly/pgloader-stat-trace-pgdata3.conf

执行加载数据

python /usr/bin/pgloader -c /export/loganalyse/pgdata3.conf

相关配置文件

[jiao@l-jiao.s.cn6 ～]$ cat pgloader-stat-trace-pgdata3.conf

[pgsql]
host = l-xxx.xx.cn5
port = 5432
base = data_sas
user = pgdba
pass = XXXXXXXXXXXXXXXXXXXXXX

log_file            = /var/pgdata3-trace-2015-12-01.log
log_min_messages    = ERROR
client_min_messages = WARNING
lc_messages         = C

client_encoding = 'utf-8'　
#input_encoding = 'utf-8'
copy_every      = 20000
commit_every    = 20000
#copy_delimiter  = %

null         = "NULL"
empty_string = ""

max_parallel_sections = 4

[pgloader]
table     = wl_data.stat_trace
format    = text
filename  = /export/data/trace.2015-12-01.log
field_sep = \t
columns   = report_date,requesttime,source,mode,uid,pids,env_type,query,pt,act,ts,platform,vid,ntitle,ntitlelen,nimg,nimglen,userid,gid
reject_log =   /export/loganalyse/tuan/pgloader-conf/log/trace-reject-2015-12-01.log
reject_data =  /export/log/trace-statistics-reject-2015-12-01.data
quotechar = ^A
escapechar = ^A