您的位置：首页 > 其它

LVS负载均衡集群搭建

2015-04-23 14:19 218 查看
环境：RHEL6-x86-64
主要软件
MySQL-5.6.14
httpd-2.4
php-5.4.13

如下图：外网用户请求到达负载调度器，根据调度算法挑选一个Real Server(后简称RS）响应，并且使用PPC（持久端口连接），将同一客户端对同一集群服务的请求始终定向至此前选定的RS。
定向到同一个RS有什么用呢？例如：访问一个电商网站，RS1响应。刷新一下网页页面，可能被定向到RS2，这样就会要求用户重新登陆，很麻烦，通过持久连接就可以解决这种问题。
还有一个问题，如何保证无论通过RS1还是RS2访问的页面都是一样的？需要实时保证web页面数据相同。这里采用inotify+sersync同步两个RS中的DocumentRoot数据，实时保持一致，也可以采用inotify+rsync解决，建议sersync解决。
用户访问页面可以使用http协议，但付款时为保证用户信息安全，需要用到https协议，这时使用PPC就不行了。要采用PNMPP（持久防火墙标记连接）将80端口和443端口绑定到一起。这样无论通过http还是https访问，都被定向到同一个RS。

这里使用LVS的IP负载均衡技术，它由IPVS模块实现，IPVS安装在Director Server（简称DS）上，在DS（这里是192.168.1.3）上虚拟一个IP（Virtual IP 简称VIP，这里是192.168.1.4），用户通过这个虚拟出来的IP访问服务器。这样用户请求通过VIP到达DS，然后DS从RS列表挑选一个RS响应。
挑选的RS如何响应用户请求呢？有三种方式VS/NAT,VS/TUN,VS/DR，这里使用VS/DR方式。




DS 192.168.1.3 VIP 192.168.1.4 配置在网卡eth0 别名eth0:0 RS1 192.168.1.5 VIP 192.168.1.4 配置在本地环回lo 别名 lo:0 RS2 192.168.1.6 VIP 192.168.1.4 配置在本地环回lo 别名 lo:0 mysql 192.168.1.7DNS 192.168.1.8
RS1上安装LAP,RS2安装LAP，mysql服务器安装在192.168.1.7上。DNS服务器搭建在192.168.1.8上。

首先搭建MySQL：

进192.168.1.7

此处用二进制免编译包
# groupadd -r -g 306 mysql
# useradd -g 306 -r -u 306 mysql
# tar xf  mysql-5.6.14-linux-glibc2.5-x86_64  /usr/local/mysql
# cd /usr/local/mysql


MySQL数据最好保存在单独分区的一个逻辑卷上，挂载至/mydata下，后在mydata目录下创建data目录，用于存放mysql数据。
磁盘分区略去，分区后如下：
# fdisk /dev/sdb
Device Boot      Start         End      Blocks   Id  System
/dev/sdb1              1        2610    20964793+  8e  Linux LVM

逻辑卷创建：
# partprobe /dev/sdb1
# pvcreate /dev/sdb1
# vgcreate myvg /dev/sdb1
# lvcreate -n mydata -L 5G myvg
# mkfs.ext4  /dev/myvg/mydata


挂载
# mkdir /mydata
# echo "/dev/myvg/mydata  /mydata ext4 defaults 0 0" >> /etc/fstab
# mount -a


创建MySQL数据目录
# mkdir /mydata/data
# chown -R mysql.mysql /mydata/data
MySQL初始化：

# cd /usr/local/mysql
# scripts/mysql_install_db --user=mysql --datadir=/mydata/data
# chown -R root /usr/local/mysql/*
提供SysV风格服务脚本
# cp support-files/mysql.server /etc/init.d/mysqld


提供MySQL配置文件
# cp my.cnf /etc/my.cnf
# vim /etc/my.cnf
basedir = /usr/local/mysql
datadir = /mydata/data
port = 3306
sock=/var/lib/mysql/mysql.sock

编辑/etc/profile.d/mysql.sh
export PATH=$PATH:/usr/local/mysql/bin


编辑/etc/ld.so.conf.d/mysql.conf
/usr/local/mysql/lib
# su
man帮助文档，编辑/etc/man.config
MANPATH /usr/local/mysql/man


头文件
ln -sv /usr/local/mysql/include/ /usr/include/mysql
# service mysqld start
# chkconfig --add mysqld
# mysql
mysql> UPDATE user SET password=PASSWORD('xiaoming') WHERE USER='root';
mysql> GRANT ALL PRIVILEGES ON *.* TO root@'192.168.1.%' IDENTIFIED BY 'xiaoming';
mysql> FLUSH PRIVILEGES;


DNS搭建

进192.168.1.8

# yum -y install bind bind-utils


配置/etc/named.conf 文件
//
// named.conf
//
// Provided by Red Hat bind package to configure the ISC BIND named(8) DNS
// server as a caching only nameserver (as a localhost DNS resolver only).
//
// See /usr/share/doc/bind*/sample/ for example named configuration files.
//

options {
directory       "/var/named";
recursion yes;

};

zone "." IN {
type hint;
file "named.ca";
};

logging {
channel query_log {
file "/var/log/named/bind_query.log" versions 3;
severity dynamic;
print-category yes;
print-time yes;
print-severity yes;
};
category queries { query_log; };
};

include "/etc/named.rfc1912.zones";
include "/etc/named.root.key";


配置/etc/named.rfc1912.zones文件
// named.rfc1912.zones:
//
// Provided by Red Hat caching-nameserver package
//
// ISC BIND named zone configuration for zones recommended by
// RFC 1912 section 4.1 : localhost TLDs and address zones
// and http://www.ietf.org/internet-drafts/draft-ietf-dnsop-default-local-zones-02.txt // (c)2007 R W Franks
//
// See /usr/share/doc/bind*/sample/ for example named configuration files.
//

zone "localhost.localdomain" IN {
type master;
file "named.localhost";
allow-update { none; };
};

zone "localhost" IN {
type master;
file "named.localhost";
allow-update { none; };
};

zone "1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa" IN {
type master;
file "named.loopback";
allow-update { none; };
};

zone "1.0.0.127.in-addr.arpa" IN {
type master;
file "named.loopback";
allow-update { none; };
};

zone "0.in-addr.arpa" IN {
type master;
file "named.empty";
allow-update { none; };
};

zone "mingxiao.info" IN {
type master;
file "mingxiao.info.zone";
};

zone "1.168.192.in-addr.arpa" IN {
type master;
file "192.168.1.zone";
};


# cd /var/named
配置mingxiao.info.zone
$TTL 86400
@ IN SOA ns.mingxiao.info admin.mingxiao.info. (
2015041211
2H
10M
3D
1D )
IN NS ns
ns IN A 192.168.1.8
www IN A 192.168.1.4


配置192.168.1.zone
$TTL 86400
@ IN SOA ns.mingxiao.info admin.mingxiao.info. (
2015041211
2H
10M
3D
1D )
IN NS ns.mingxiao.info.
8     IN PTR ns.mingxiao.info.
4     IN PTR www.mingxiao.info.
# chown root:named mingxiao.info.zone 192.168.1.zone
# chmod 640 mingxiao.info.zone 192.168.1.zone


各主机中/etc/resolv.conf配置，DNS都指向192.168.1.8主机
nameserver 192.168.1.8
#service named start


LAP搭建
进192.168.1.5

安装Apache

1，安装apr
# cd /usr/local
# tar xf apr-1.5.1.tar.bz2
# cd apr-1.5.1
# ./configure --prefix=/usr/local/apr
# make
# make install


2,安装apr-util
# ./configure --prefix=/usr/local/apr-util --with-apr=/usr/local/apr
# make
# make install


3,安装httpd
# tar xf httpd-2.4.10.tar.bz2
# cd httpd-2.4.10


# ./configure \
--prefix=/usr/local/apache \
--sysconfdir=/etc/httpd \
--enable-so \
--enable-rewrite \
--enable-ssl     \
--enable-cgi \
--enable-cgid \
--enable-modules=most \
--enable-mods-shared=most \
--enable-mpms-shared=all \
--with-mpm=event \
--with-apr=/usr/local/apr \
--with-apr-util=/usr/local/apr-util \
出现错误：
configure: error: pcre-config for libpcre not found. PCRE is required anavailabl from
解决办法：
# yum install -y pcre-devel
# make
# make install
编辑/etc/httpd/httpd.conf,加入
PidFile "/var/run/httpd.pid"


提供SysV风格的启动脚本：/etc/init.d/httpd
#!/bin/bash
#
# httpd        Startup script for the Apache HTTP Server
#
# chkconfig: - 85 15
# description: Apache is a World Wide Web server.  It is used to serve \
#       HTML files and CGI.
# processname: httpd
# config: /etc/httpd/conf/httpd.conf
# config: /etc/sysconfig/httpd
# pidfile: /var/run/httpd.pid
# Source function library.
. /etc/rc.d/init.d/functions
if [ -f /etc/sysconfig/httpd ]; then
. /etc/sysconfig/httpd
fi
# Start httpd in the C locale by default.
HTTPD_LANG=${HTTPD_LANG-"C"}
# This will prevent initlog from swallowing up a pass-phrase prompt if
# mod_ssl needs a pass-phrase from the user.
INITLOG_ARGS=""
# Set HTTPD=/usr/sbin/httpd.worker in /etc/sysconfig/httpd to use a server
# with the thread-based "worker" MPM; BE WARNED that some modules may not
# work correctly with a thread-based MPM; notably PHP will refuse to start.
# Path to the apachectl script, server binary, and short-form for messages.
apachectl=/usr/local/apache/bin/apachectl
httpd=${HTTPD-/usr/local/apache/bin/httpd}
prog=httpd
pidfile=${PIDFILE-/var/run/httpd.pid}
lockfile=${LOCKFILE-/var/lock/subsys/httpd}
RETVAL=0
start() {
echo -n $"Starting $prog: "
LANG=$HTTPD_LANG daemon --pidfile=${pidfile} $httpd $OPTIONS
RETVAL=$?
echo
[ $RETVAL = 0 ] && touch ${lockfile}
return $RETVAL
}
stop() {
echo -n $"Stopping $prog: "
killproc -p ${pidfile} -d 10 $httpd
RETVAL=$?
echo
[ $RETVAL = 0 ] && rm -f ${lockfile} ${pidfile}
}
reload() {
echo -n $"Reloading $prog: "
if ! LANG=$HTTPD_LANG $httpd $OPTIONS -t >&/dev/null; then
RETVAL=$?
echo $"not reloading due to configuration syntax error"
failure $"not reloading $httpd due to configuration syntax error"
else
killproc -p ${pidfile} $httpd -HUP
RETVAL=$?
fi
echo
}
# See how we were called.
case "$1" in
start)
start
;;
stop)
stop
;;
status)
status -p ${pidfile} $httpd
RETVAL=$?
;;
restart)
stop
start
;;
condrestart)
if [ -f ${pidfile} ] ; then
stop
start
fi
;;
reload)
reload
;;
graceful|help|configtest|fullstatus)
$apachectl $@
RETVAL=$?
;;
*)
echo $"Usage: $prog {start|stop|restart|condrestart|reload|status|fullstatus|graceful|help|configtest}"
exit 1
esac
exit $RETVAL


# chmod +x /etc/init.d/httpd
# chkconfig --add httpd


编辑/etc/profile.d/httpd.sh
PATH=$PATH:/usr/local/apache/bin
# source /etc/profile.d/httpd.sh
#  service httpd start


安装php
# cd php-5.4.13
# ./configure \
--prefix=/usr/local/php \
--with-mysql=mysqlnd \
--with-mysqli=mysqlnd \
--with-pdo-mysql=mysqlnd \
--with-openssl \
--enable-mbstring \
--with-freetype-dir \
--with-jpeg-dir \
--with-png-dir \
--with-zlib \
--with-libxml-dir=/usr \
--enable-xml \
--enable-sockets \
--with-apxs2=/usr/local/apache/bin/apxs \
--with-mcrypt --with-config-file-path=/etc \
--with-config-file-scan-dir=/etc/php.d \
--with-bz2 --enable-maintainer-zts \

出现错误：
configure: error: mcrypt.h not found. Please reinstall libmcrypt.
解决办法：
#yum -y install libmcrypt-devel mhash-devel
# make
# make install


提供php配置文件
cp php.ini-production /etc/php.ini


/etc/httpd/httpd.conf配置文件
AddType application/x-httpd-php .php
AddType application/x-httpd-php-source .phps
DirectoryIndex index.php index.html
注释掉
#/DocumentRoot "/usr/local/apache/htdocs"
取消注释
Include /etc/httpd/extra/httpd-vhosts.conf


/etc/httpd/extra/httpd-vhosts.conf配置
<VirtualHost *:80>
DocumentRoot "/www/mingxiao"
<Directory "/www/mingxiao">
Options none
AllowOverride none
Require all granted
</Directory>
ServerName www.mingxiao.info
ErrorLog "/var/log/httpd/mingxiao.info_error.log"
CustomLog "/var/log/httpd/mingxiao.info_access.log" combined
</VirtualHost>
# mkdir -pv /www/mingxiao
# mkdir /var/log/httpd
# service httpd restart

进192.168.1.6
安装LAP，操作步骤同192.168.1.5。

192.168.1.5和192.168.1.6安装完LAP以后，在192.168.1.5主机安装Discuz论坛
# unzip Discuz_X3.2_SC_UTF8.zip
# cd upload/
# cp -R ./* /www/mingxiao


接下来想要保持192.168.1.5和192.168.1.6中DocumentRoot数据相同
这里采用inotify+sersync解决。

进192.168.1.5
安装sersync和inotify
# yum install inotify-tools
# tar xf sersync2.5.4_64bit_binary_stable_final.tar.gz -C /usr/local/sersync
# cd /usr/local/sersync


编辑confxml.xml
<head version="2.5">
<host hostip="192.168.1.5" port="8008"></host>
.......
<sersync>
<localpath watch="/www/mingxiao">
<remote ip="192.168.1.6" name="xiaoming"/>
</localpath>
<rsync>
<commonParams params="-zrtopg"/>
<userDefinedPort start="false" port="874"/><!-- port=874 -->
<auth start="true" users="xiaoming" passwordfile="/etc/rsync.pass"/>
</rsync>
<failLog path="/tmp/rsync_fail_log.sh" timeToExecute="60"/><!--default every 60mins execute once-->
<crontab start="true" schedule="300"><!--600mins-->
.......


编辑 /etc/rsync.pass，添加以下内容
xiaoming
#chmod 600 /etc/rsync.pass


进192.168.1.6
# useraddxiaoming
# echoxiaoming | passwd --stdinxiaoming


为rsync提供配置文件，编辑/etc/rsyncd.conf
uid=nobody
gid=nobody
use chroot = no
max connections =10
strict modes = yes
pid file = /var/run/rsyncd.pid
lock file = /var/run/rsync.lock
log file = /var/log/rsyncd.log
[xiaoming]
path = /www/mingxiao
comment =xiaoming file
ignore errors
read only = no
write only = no
host allow = 192.168.1.5
list = false
uid = root
gid = root
auth user =xiaoming
secrets file = /etc/server.pass


编辑/etc/server.pass
xiaoming
# chmod 600 /etc/server.pass
#chkconfig rsync on
# service xinetd start


进Director 192.168.1.3
# echo 1 > /proc/sys/net/ipv4/ip_forward
# ifconfig eth0:0 192.168.1.4
# route add –host 192.168.1.4 dev eth0:0


进RS1 192.168.1.5
# ifconfig lo:0 192.168.1.4 broadcast 192.168.1.4 netmask 255.255.255.255 up
# route add –host 192.168.1.4 dev lo:0
# echo 1 > /proc/sys/net/ipv4/conf/eth0/arp_ignore
# echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
#　echo 2 > /proc/sys/net/ipv4/conf/eth0/arp_announce
＃　echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce


进RS1 192.168.1.6
＃　ifconfig lo:0 192.168.1.4 broadcast 192.1.4 netmask 255.255.255.255 up
# route add –host 192.168.1.4 dev lo:0
# echo 1 > /proc/sys/net/ipv4/conf/eth0/arp_ignore
#　echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
# echo 2 > /proc/sys/net/ipv4/conf/eth0/arp_announce
# echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce


进Director 192.168.1.3
Ipvsadm –A –t 192.168.1.4:80  -s rr
Ipvsadm –a –t 192.168.1.4:80 –r 192.168.1.5  –g –w 2
Ipvsadm –a –t 192.168.1.4:80 –r 192.168.1.6  –g –w 1


进192.168.1.5

#  /usr/local/sersync/sersync2 -r -o /usr/local/sersync/confxml.xml -n 5 -d
可以看到192.168.1.5和192.168.1.6的DoucumentRoot文件已经同步，保持了一致。

通过浏览器访问www.mingxiao.info轮流定向到192.168.1.5和192.168.1.6这两个RS上。

接下来配置使用https
模拟192.168.1.8为证书颁发机构，

进192.168.1.8
#cd /etc/pki/CA


生成私钥
#（umask 077; openssl genrsa -out private/cakey.pem 2048)


编辑 /etc/pki/tls/openssl.cnf
[ CA_default ]
dir             = /etc/pki/CA           # Where everything is kept
certs           = $dir/certs            # Where the issued certs are kept
crl_dir         = $dir/crl              # Where the issued crl are kept
database        = $dir/index.txt        # database index file.
#unique_subject = no                    # Set to 'no' to allow creation of
# several ctificates with same subject.
new_certs_dir   = $dir/newcerts         # default place for new certs.
certificate     = $dir/cacert.pem       # The CA certificate
serial          = $dir/serial           # The current serial number
crlnumber       = $dir/crlnumber        # the current crl number
# must be commented out to leave a V1 CRL
crl             = $dir/crl.pem          # The current CRL
private_key     = $dir/private/cakey.pem# The private key
RANDFILE        = $dir/private/.rand
[ req_distinguished_name ]
countryName                     = Country Name (2 letter code)
countryName_default             = CN
stateOrProvinceName             = State or Province Name (full name)
stateOrProvinceName_default     = HeNan
localityName                    = Locality Name (eg, city)
localityName_default    = AnYang
0.organizationName              = Organization Name (eg, company)
0.organizationName_default      = XiaoMing
organizationalUnitName          = Organizational Unit Name (eg, section)
organizationalUnitName_default  = Tech


生成自签证书
# openssl req -new -x509 -key private/cakey.pem -out cacert.pem -days 3655
这里会让输入一些有关证书的信息，如国家，省份等，以刚才在openssl.cnf配置中填写的默认选项，直接敲回车即可，下两项随意写了。
Common name :  ca.mingxiao.info
Email Address :   ca@mingxiao.info
# touch index.txt
# touch serial
# echo 01 > serial


进192.168.1.5
生成证书颁发请求，然后到192.168.1.8签署证书。

复制192.168.1.8上的配置文件，省去重复配置
# scp 192.168.1.109:/etc/pki/tls/openssl.cnf /etc/pki/tls/openssl.cnf
# mkdir /etc/httpd/ssl
# cd /etc/httpd/ssl
# (umask 077; openssl genrsa 2048 > httpd.key)
# openssl req -new -key httpd.key -out  httpd.csr
Common name www.mingxiao.info   这一项最关键，必须是httpd中ServerName的名字，否则会有错。
Email Addressxiaoming@mingxiao.info

准备签证，将签证请求文件复制到有签证权利的那台机器
# scp httpd.csr 192.168.1.8:/tmp


进192.168.1.8
进行签证：
# openssl ca -in /tmp/httpd.csr -out /tmp/httpd.crt -days 3655


进192.168.1.5
将证书复制过来
scp 192.168.1.109:/tmp/httpd.crt /etc/httpd/ssl/httpd.crt


编辑/etc/httpd/httpd.conf
取消下几项注释
LoadModule socache_shmcb_module modules/mod_socache_shmcb.so
LoadModule ssl_module modules/mod_ssl.so
Include /etc/httpd/extra/httpd-ssl.conf


编辑/etc/httpd/extra/httpd-ssl.conf
<VirtualHost 192.168.1.4:443>
ServerName www.mingxiao.info
DocumentRoot "/www/mingxiao"
<Directory "/www/mingxiao">
Options none
AllowOverride none
Require all granted
</Directory>
ErrorLog "/usr/local/apache/logs/error_log"
TransferLog "/usr/local/apache/logs/access_log"
...........
#   SSL Engine Switch:
#   Enable/Disable SSL for this virtual host.
SSLEngine on
SSLCertificateKeyFile "/etc/httpd/ssl/httpd.key"
SSLCertificateFile "/etc/httpd/ssl/httpd.crt"


在192.168.1.16上可以直接用192.168.1.5的证书，因此
进入192.168.1.108
# mkdir -pv /etc/httpd/ssl
# scp -r 192.168.1.5:/etc/httpd/ssl/ /etc/httpd/
直接使用192.168.1.5的httpd配置文件
# scp 192.168.1.5:/etc/httpd/httpd.conf /etc/httpd/httpd.conf
# scp 192.168.1.5:/etc/httpd/extar/httpd-ssl.conf /etc/httpd/extar/httpd-ssl.conf


证书签署完毕。

进192.168.1.3
接下来使用PNMPP，使用持久防火墙标记连接，将80端口和443端口绑定到一起
# ipvsadm -C
# iptables -t mangle -A PREROUTING -d 192.168.1.4 -i eth0 -p tcp --dport 80 -j MARK --set-mark 8
# iptables -t mangle -A PREROUTING -d 192.168.1.4 -i eth0 -p tcp --dport 443 -j MARK --set-mark 8
# ipvsadm -A -f 8 -s rr -p 600
# ipvsadm -a -f 8 -r 192.168.1.5 -g -w 1
# ipvsadm -a -f 8 -r 192.168.1.6 -g -w 2


进192.168.1.5
# service httpd restart
进192.168.1.6
# service httpd restart


一切大工告成，然后将证书拷到windowns主机一份，安装证书。
浏览器输入http://www.mingxiao.info https://www.mingxiao.info 访问即可。
# ipvsadm -L -n -c
IPVS connection entries
pro expire state             source          virtual           destination
TCP 01:47  FIN_WAIT    192.168.1.105:55683 192.168.1.4:443    192.168.1.6:443
TCP 01:47  FIN_WAIT    192.168.1.105:55685 192.168.1.4:443    192.168.1.6:443
TCP 01:33  FIN_WAIT    192.168.1.105:55654 192.168.1.4:443    192.168.1.6:443
TCP 00:51  FIN_WAIT    192.168.1.105:55518 192.168.1.4:80     192.168.1.6:80
TCP 01:33  FIN_WAIT    192.168.1.105:55647 192.168.1.4:443    192.168.1.6:443
TCP 00:07  CLOSE       192.168.1.128:1320 192.168.1.4:80     192.168.1.5:80
TCP 00:07  CLOSE       192.168.1.128:1321 192.168.1.4:80     192.168.1.5:80
TCP 00:07  CLOSE       192.168.1.128:1323 192.168.1.4:80     192.168.1.5:80
TCP 00:07  CLOSE       192.168.1.128:1317 192.168.1.4:80     192.168.1.5:80
TCP 00:07  CLOSE       192.168.1.128:1319 192.168.1.4:80     192.168.1.5:80
TCP 00:07  CLOSE       192.168.1.128:1322 192.168.1.4:80     192.168.1.5:80
可以看到同一主机不论是通过http还是https访问都被定向到同一RS。不同主机访问被定向到不同RS。
本文出自 “在路上” 博客，转载请与作者联系！
内容来自用户分享和网络整理，不保证内容的准确性，如有侵权内容，可联系管理员处理 点击这里给我发消息
标签： 
相关文章推荐
新的分享
章节导航