nginx+keepalived实战高可用反向代理及nginx参数优化

一、nginx编译安装(2台server都安装nginx)

创建nginx单独的运行用户nginx：

# sudo useradd -r nginx

安装依赖包文件：

# sudo yum install pcre pcre-devel openssl-devel

利用TCMalloc优化Nginx性能:

TCMalloc(Thred-Caching Malloc)是谷歌开发的开源工具 “google-perftools”中的一个成员。与标准glibc库的maclloc相比，TCMalloc库在内存分配速率和速度上要高很多，
这在很大程度上提高了服务器在高并发情况下的性能，从而降低系统负载。
安装TCMalloc库，需要安装libunwind(32bit os 不要安装)和google-perftools两个软件包，libunwind库为64bitCPU和操作系统的程序提供了基本函数调用链和函数调用寄存
器功能。

安装libunwind库：
下载地址：--http://download.savannah.gnu.org/releases/libunwind/ 安装：

# sudo tar zxvf libunwind-1.1.tar.gz
# cd libunwind-1.1
# sudo CFLAGS=-fPIC ./configure
# sudo make CFLAGS=-fPIC
# sudo make CFLAGS=-fPIC install

安装google-perftools:
下载地址：--[http://code.google.com/p/gperftools/ ]

# sudo tar zxvf gperftools-2.3.tar.gz
# cd gperftools-2.3
# sudo ./configure
# sudo make && make install
# sudo sh -c 'echo "/usr/local/lib" > /etc/ld.so.conf.d/usr_local_lib.conf'
# sudo ldconfig

安装配置Nginx：
取消Debug模式
在解压Nginx源码文件后，找到源码目录下的auto/cc/gcc文件，打开后在其中找到如下：

# debug
CFLAGS="$CFLAGS -g"

修改为：

# debug
# CFLAGS="$CFLAGS -g"

编译安装Nginx：

# sudo ./configure --user=nginx --group=nginx --prefix=/usr/local/nginx-1.6.2 --with-http_stub_status_module --with-http_ssl_module --with-http_gzip_static_module --with-pcre --with-http_realip_module --with-google_perftools_module --with-cc-opt='-O3'
# sudo make
# sudo make install

为google-perftools添加线程目录：

# mkdir /tmp/tcmalloc
# chmod 0777 /tmp/tcmalloc/

修改nginx配置文件：

在pid这行下添加：
#pid        logs/nginx.pid;
google_perftools_profiles /tmp/tcmalloc;

重启Nginx，完成google-perftools的加载。
验证运行状态：

[cpis@varnish01 conf]$ sudo /usr/local/nginx-1.6.2/sbin/nginx -c /usr/local/nginx-1.6.2/conf/nginx.conf
[cpis@varnish01 conf]$ sudo lsof -n|sudo grep tcmalloc
nginx     31327   nginx    9w      REG                8,3        0    2359363 /tmp/tcmalloc.31327
nginx     31328   nginx   11w      REG                8,3        0    2359364 /tmp/tcmalloc.31328
nginx     31329   nginx   13w      REG                8,3        0    2359365 /tmp/tcmalloc.31329
nginx     31330   nginx   15w      REG                8,3        0    2359367 /tmp/tcmalloc.31330

至此利用TcMalloc优化nginx的操作完成。
nginx内核参数优化：

net.ipv4.tcp_max_tw_buckets = 6000
net.ipv4.ip_local_port_range = 1025 65000
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_syncookies = 1
net.core.somaxconn = 262144
net.core.netdev_max_backlog = 262144
net.ipv4.tcp_max_orphans = 262144
net.ipv4.tcp_max_syn_backlog = 262144
net.ipv4.tcp_synack_retries = 1
net.ipv4.tcp_syn_retries = 1
net.ipv4.tcp_fin_timeout = 1
net.ipv4.tcp_keepalive_time = 30

二、nginx配置参数优化
修改配置文件：

# vi /usr/loca/nginx-1.6.2/conf/nginx.conf
user  nginx nginx;
worker_processes  auto;

events {
worker_connections  1024;
use epoll;
}
proxy_buffering off;
proxy_connect_timeout       3s;
proxy_ignore_client_abort on;
proxy_read_timeout          120s;
proxy_send_timeout          120s;
sendfile        on;
tcp_nopush     on;
client_max_body_size 20m;
keepalive_timeout  120;
client_body_buffer_size  1024k;
proxy_buffer_size        64k;
proxy_buffers            32 32k;
proxy_busy_buffers_size 128k;
proxy_temp_file_write_size 64k;
gzip  on;
gzip_min_length  1000;
gzip_buffers     4 8k;
gzip_types       text/plain application/x-javascript text/css application/xml text/javascript;
gzip_vary on;

三、配置HA(两台server都安装keepalived)
3.1 server1 配置
# yum install keepalived -y
# vi /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
sysadmin@fhy.com
}
notification_email_from monitor@fhy.com
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id LVS_nginx
}
vrrp_instance HA_1 {
state BACKUP
interface eth0
virtual_router_id 80
priority 100 #优先级
advert_int 2
nopreempt #不抢占
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
172.31.24.200
}
}
virtual_server 172.31.24.200 80 {
delay_loop 2
lb_algo wlc
lb_kind DR
nat_mask 255.255.255.0
protocol TCP
persistence_timeout 60
real_server 172.31.24.62 80 {
weight 1
notify_down /etc/keepalived/nginxcheck/nginx.sh #nginx挂了触发脚本，杀掉进程keepalived，使得vip漂移到另外一台server,实现高可用
TCP_CHECK {
connect_port 80
connect_timeout 3
nb_get_retry 2
delay_before_retry 1
}
}
}
nginx.sh脚本
# mkdir -p /etc/keepalived/nginxcheck
# cd /etc/keepalived/nginxcheck
# cat > nginx.sh << EOF
#! /bin/bash
pkill keepalived
EOF
# chmod u+x nginx.sh
3.2 server2 配置
# yum install keepalived -y
# vi /etc/keepalived/keepalived.conf! Configuration File for keepalivedglobal_defs { notification_email { sysadmin@fhy.com } notification_email_from monitor@fhy.com smtp_server 127.0.0.1 smtp_connect_timeout 30 router_id LVS_nginx}vrrp_instance HA_1 { state BACKUP interface eth0 virtual_router_id 80 priority 90 #优先级 advert_int 2 nopreempt #不抢占 authentication { auth_type PASS auth_pass 1111 } virtual_ipaddress { 172.31.24.200 }}virtual_server 172.31.24.200 80 { delay_loop 2 lb_algo wlc lb_kind DR nat_mask 255.255.255.0 protocol TCP persistence_timeout 60 real_server 172.31.24.66 80 { weight 1 notify_down /etc/keepalived/nginxcheck/nginx.sh #nginx挂了触发脚本，杀掉进程keepalived，使得vip漂移到另外一台server,实现高可用 TCP_CHECK { connect_port 80 connect_timeout 3 nb_get_retry 2 delay_before_retry 1 } }}nginx.sh脚本# mkdir -p /etc/keepalived/nginxcheck# cd /etc/keepalived/nginxcheck# cat > nginx.sh << EOF#! /bin/bashpkill keepalivedEOF# chmod u+x nginx.sh四、启动服务
两台server均启动keepalived和nginx进程，测试的时候，启停nginx
# tail -f /var/log/messages (观察vip的漂移情况)
本文出自 “hanyun.fang” 博客，转载请与作者联系！