Linux LVS高并发测试程序，内核参数设置，连接数查询指令

最近在公司参与了一个LVS系统测试的项目，学习到了一些关于高并发测试相关的知识，写到博客里记录下

Linux内核参数设置

在服务器端需要调整系统最大文件句柄数

ulimit -n 1000000

在服务器硬件支持，以及服务较轻量的情况下，最大连接数未必只限于100000,视情况而定吧

客户端需要发起大量连接，理论上是最多可支持65535个连接，但实际上Linux的默认可用端口号并没有这么多，需要修改

sudo echo 1024 65534 > /proc/sys/net/ipv4/ip_local_port_range

查看连接个数

我们一开始用的是

netstat -nap | grep 程序名 | grep ESTABLISHED| wc -l

来查询，这个指令当连接数上万后就变得很慢，而且我们发现了一个略坑的问题，通过这组合指令查询到的结果是波动的，我特意试了10000个连接下，在程序没有报有连接关闭的情况下，通过netstat查出来的连接个数在9998到10000之间波动。最后我换了个指令

ss state ESTABLISHED dport = :端口号 and dst IP | wc -l

通过ss指令查询，不仅速度很快，而且不会出现计数不准的情况，需要注意的是，查询到的连接数要比实际多1

测试程序

/*client1.cpp
*该程序需要libevent库支持
*测试有无LVS时连接延时区别的程序
*程序发起指定数量的连接后，将每秒给服务器发送一个报文，每个连接收到
*服务器返回的报文后计算耗时
*/
#include <stdlib.h>
#include <stdio.h>
#include <errno.h>
#include <string.h>
#include <unistd.h>
#include <netdb.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <sys/types.h>
#include <arpa/inet.h>
#include <sys/time.h>

#include <iostream>
#include <map>
#include <vector>
#include <string>

#include "event2/bufferevent.h"
#include "event2/buffer.h"
#include "event2/listener.h"
#include "event2/util.h"
#include "event2/event.h"

using namespace std;

typedef struct bufferevent* BufferEvent;

map<struct bufferevent*, long long>    bufferevent_map;
map<struct bufferevent*, long long>::iterator  iter;
static int      index_ = 0;

struct cb_arg
{
struct event *ev;
struct timeval tv;
};

void timeout_cb(int fd, short event, void *params)
{
if (iter == bufferevent_map.end()) {
return ;
}

char buf[1024];
struct cb_arg *arg = (struct cb_arg*)params;
struct event *timeout = arg->ev;
struct timeval tv = arg->tv;
struct timeval start;

struct bufferevent *bufevt = iter->first;
gettimeofday(&start,NULL);
iter->second =  1000000 * start.tv_sec + start.tv_usec;
sprintf(buf, "%lld", iter->second);
bufferevent_write(bufevt, buf, strlen(buf));
printf("fd %u send %s\n", fd, buf);
index_++;
iter++;

evtimer_add(timeout, &tv);
}

void read_cb(struct bufferevent *bufevt, void *arg)
{
char    line[1024] = {0};
size_t     n;
evutil_socket_t fd = bufferevent_getfd(bufevt);
struct timeval start;
map<struct bufferevent*, long long>::iterator it_temp;

it_temp = bufferevent_map.find(bufevt);
if (it_temp == bufferevent_map.end()) {

return ;
}

gettimeofday(&start,NULL);
long long now =  1000000 * start.tv_sec + start.tv_usec;
long long used = now - it_temp->second;

while (n = bufferevent_read(bufevt, line, 1024), n > 0)
{
printf("fd=%u, used time: %lld us\n", fd, used);
}
}

void error_cb(struct bufferevent *bufevt, short events, void *user_data)
{
evutil_socket_t fd = bufferevent_getfd(bufevt);

if (events & BEV_EVENT_EOF)
{
printf("Connection closed.\n");
printf("fd %u close !\n", fd);
bufferevent_free(bufevt);
}
else if (events & BEV_EVENT_ERROR)
{
printf("Got an error on the connection: %s\n", strerror(errno));
printf("fd %u close !\n", fd);
bufferevent_free(bufevt);
}
}

int main(int argc, char *argv[])
{
if (argc != 4) {
printf("参数个数不正确 %d, 需要3个参数，连接个数，IP，端口号\n", argc - 1);
return 0;
}

vector<string>  argvs;
for (int i = 1; i < argc; ++i) {
argvs.push_back(argv[i]);
}

timeval     tv = {1, 0};
int         times = atoi(argvs[0].c_str());
int         port = atoi(argvs[2].c_str());
struct sockaddr_in  sin;
struct bufferevent* bufevt;

event_base  *evbase = event_base_new();
if (evbase == NULL) {
printf("err in new event_base\n");
return 0;
}

struct event*   timeout;
struct cb_arg   arg;
timeout = evtimer_new(evbase, timeout_cb, &arg);
if (timeout == NULL) {
printf("err in new evtimer\n");
return 0;
}
arg.ev = timeout;
arg.tv = tv;

memset(&sin, 0, sizeof(sin));
sin.sin_family = AF_INET;
sin.sin_addr.s_addr = inet_addr(argvs[1].c_str());
sin.sin_port = htons(port);
memset(sin.sin_zero, 0x00, 8);

for (int i = 0; i < times; ++i) {
int fd = socket(AF_INET, SOCK_STREAM, 0);
evutil_make_socket_nonblocking(fd);

bufevt = bufferevent_socket_new(evbase, fd, BEV_OPT_CLOSE_ON_FREE);
if (bufevt == NULL) {
printf("err in new bufferevent, fd %u connect failed !\n", fd);
continue ;
}

bufferevent_setcb(bufevt, read_cb, NULL, error_cb, NULL);
bufferevent_enable(bufevt, EV_READ | EV_WRITE | EV_PERSIST);

if (bufferevent_socket_connect(bufevt, (struct sockaddr *)&sin, sizeof(sin)) < 0) {
printf("err in connect with server, fd %u connect failed !\n", fd);
bufferevent_free(bufevt);
continue ;
}
bufferevent_map.insert(make_pair(bufevt, 0));
}

iter = bufferevent_map.begin();

evtimer_add(timeout, &tv);
event_base_dispatch(evbase);
evtimer_del(timeout);
event_base_free(evbase);

return 0;
}

/*server.cpp
*该程序需要libevent库支持
*简单的回射服务器
*/
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <assert.h>

#include <event2/event.h>
#include <event2/bufferevent.h>

#define LISTEN_BACKLOG 1024

static int time_out_count = 0;
static int err_conn_count = 0;
static int err_accept_count = 0;

void do_accept(evutil_socket_t listener, short event, void *arg);
void read_cb(struct bufferevent *bev, void *arg);
void error_cb(struct bufferevent *bev, short event, void *arg);
void write_cb(struct bufferevent *bev, void *arg);

int main(int argc, char *argv[])
{
if (argc != 2) {
printf("参数个数不正确 %d\n", argc);
return 0;
}

int port = atoi(argv[1]);
evutil_socket_t listener;
listener = socket(AF_INET, SOCK_STREAM, 0);
assert(listener > 0);
evutil_make_listen_socket_reuseable(listener);

struct sockaddr_in sin;
sin.sin_family = AF_INET;
sin.sin_addr.s_addr = 0;
sin.sin_port = htons(port);

if (bind(listener, (struct sockaddr *)&sin, sizeof(sin)) < 0) {
perror("bind");
return 1;
}

if (listen(listener, LISTEN_BACKLOG) < 0) {
perror("listen");
return 1;
}

printf ("Listening...\n");

evutil_make_socket_nonblocking(listener);

struct event_base *base = event_base_new();
assert(base != NULL);
struct event *listen_event;
listen_event = event_new(base, listener, EV_READ | EV_PERSIST, do_accept, (void*)base);
event_add(listen_event, NULL);
event_base_dispatch(base);

printf("The End.");
return 0;
}

void do_accept(evutil_socket_t listener, short event, void *arg)
{
struct event_base *base = (struct event_base *)arg;
evutil_socket_t fd;
struct sockaddr_in sin;
socklen_t slen;

fd = accept(listener, (struct sockaddr *)&sin, &slen);
evutil_make_socket_nonblocking(fd);
if (fd < 0) {
printf("err in accept, err ccode %d, 发生accept错误的个数 %d", errno, ++err_accept_count);
return;
}

printf("ACCEPT: fd = %u\n", fd);

struct bufferevent *bev = bufferevent_socket_new(base, fd, BEV_OPT_CLOSE_ON_FREE);
bufferevent_setcb(bev, read_cb, NULL, error_cb, arg);
bufferevent_enable(bev, EV_READ | EV_WRITE | EV_PERSIST);
}

void read_cb(struct bufferevent *bev, void *arg)
{
#define MAX_LINE    256
char line[MAX_LINE+1];
int n;
evutil_socket_t fd = bufferevent_getfd(bev);

while (n = bufferevent_read(bev, line, MAX_LINE), n > 0) {
line
= '\0';
bufferevent_write(bev, line, n);
printf("recv : %s\n", line);
}
}

void write_cb(struct bufferevent *bev, void *arg) {}

void error_cb(struct bufferevent *bev, short event, void *arg)
{
evutil_socket_t fd = bufferevent_getfd(bev);
printf("fd = %u, ", fd);
if (event & BEV_EVENT_TIMEOUT) {
time_out_count++;
printf("Timed out, 发生超时连接个数 %d", time_out_count);
printf(", err code %d\n", errno);
}
else if (event & BEV_EVENT_EOF) {
printf("connection closed\n");
}
else if (event & BEV_EVENT_ERROR) {
err_conn_count++;
printf("some other error, 连接被异常关闭个数 %d", err_conn_count);
printf(", err code %d\n", errno);
}
bufferevent_free(bev);
}

/*client2.cpp
*该程序需要libevent库支持
*测试高并发量的测试客户端程序，该程序每隔10s就会通过所有连接向服务
*器发送一个报文，以免长时间没有交互，连接被LVS断掉，同时测试服务器
*的负载
*/

#include <stdlib.h>
#include <stdio.h>
#include <errno.h>
#include <string.h>
#include <unistd.h>
#include <netdb.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <sys/types.h>
#include <arpa/inet.h>
#include <sys/time.h>

#include <iostream>
#include <map>
#include <vector>
#include <string>

#include "event2/bufferevent.h"
#include "event2/buffer.h"
#include "event2/listener.h"
#include "event2/util.h"
#include "event2/event.h"

using namespace std;

typedef struct bufferevent* BufferEvent;

map<struct bufferevent*, long long>    bufferevent_map;
map<struct bufferevent*, long long>::iterator  iter;
FILE*           fp = NULL;
static int      index_ = 0;
static int      counter = 0;
static int      begin_time = 0;

struct cb_arg
{
struct event *ev;
struct timeval tv;
};

void timeout_cb(int fd, short event, void *params)
{
char buf[1024];
struct cb_arg *arg = (struct cb_arg*)params;
struct event *timeout = arg->ev;
struct timeval tv = arg->tv;

for (iter = bufferevent_map.begin(); iter != bufferevent_map.end(); ++iter) {
memset(buf, 0, sizeof(buf));
struct bufferevent *bufevt = iter->first;

sprintf(buf, "%lld", iter->second);
bufferevent_write(bufevt, buf, strlen(buf));
}

evtimer_add(timeout, &tv);
}

void read_cb(struct bufferevent *bufevt, void *arg)
{
char    line[1024] = {0};
size_t     n;
evutil_socket_t fd = bufferevent_getfd(bufevt);
struct timeval start;
map<struct bufferevent*, long long>::iterator it_temp;

it_temp = bufferevent_map.find(bufevt);
if (it_temp == bufferevent_map.end()) {

return ;
}

while (n = bufferevent_read(bufevt, line, 1024), n > 0)
{
;
}
}

void write_count(int count)
{
char  buf[1024] = {0};
int   now_time = (int)time(NULL);

sprintf(buf, "pass %d s, now connect count %d\n", now_time - begin_time, count);
fputs(buf, fp);
}

void error_cb(struct bufferevent *bufevt, short events, void *user_data)
{
evutil_socket_t fd = bufferevent_getfd(bufevt);

if (events & BEV_EVENT_EOF)
{
counter--;
printf("Connection closed.\n");
printf("fd %u close ! count %d\n", fd , counter);
printf("now EST connect count %d\n", counter);
bufferevent_free(bufevt);
bufferevent_map.erase(bufevt);
write_count(counter);
}
else if (events & BEV_EVENT_ERROR)
{
counter--;
printf("Got an error on the connection: %d\n", errno);
printf("fd %u close ! count %d\n", fd , counter);
printf("now EST connect count %d\n", counter);
bufferevent_free(bufevt);
bufferevent_map.erase(bufevt);
write_count(counter);
}
}

void create_file()
{
char  buf[1024] = {0};
timeval tv;
gettimeofday(&tv, NULL);

int   time_ = (int) (1000000 * tv.tv_sec + tv.tv_usec);

sprintf(buf, "result_%d.txt", time_);

fp = fopen(buf, "w+");
if (fp == NULL) {
printf("create result file failed!\n");
exit(-1);
}
}

int main(int argc, char *argv[])
{
if (argc != 4) {
printf("参数个数不正确 %d, 需要3个参数，连接个数，IP，端口号\n", argc - 1);
return 0;
}

vector<string>  argvs;
for (int i = 1; i < argc; ++i) {
argvs.push_back(argv[i]);
}

create_file();

timeval     tv = {10, 0};
int         times = atoi(argvs[0].c_str());
int         port = atoi(argvs[2].c_str());
struct sockaddr_in  sin;
struct bufferevent* bufevt;

event_base  *evbase = event_base_new();
if (evbase == NULL) {
printf("err %d in new event_base\n", errno);
return 0;
}

struct event*   timeout;
struct cb_arg   arg;
timeout = evtimer_new(evbase, timeout_cb, &arg);
if (timeout == NULL) {
printf("err in new evtimer\n");
return 0;
}
arg.ev = timeout;
arg.tv = tv;

memset(&sin, 0, sizeof(sin));
sin.sin_family = AF_INET;
sin.sin_addr.s_addr = inet_addr(argvs[1].c_str());
sin.sin_port = htons(port);
memset(sin.sin_zero, 0x00, 8);

for (int i = 0; i < times; ++i) {
int fd = socket(AF_INET, SOCK_STREAM, 0);
evutil_make_socket_nonblocking(fd);
bufevt = bufferevent_socket_new(evbase, fd, BEV_OPT_CLOSE_ON_FREE);
if (bufevt == NULL) {
printf("err %d in new bufferevent, fd %u connect failed !\n", fd, errno);
continue ;
}

bufferevent_setcb(bufevt, read_cb, NULL, error_cb, NULL);
bufferevent_enable(bufevt, EV_READ | EV_WRITE | EV_PERSIST);

if (bufferevent_socket_connect(bufevt, (struct sockaddr *)&sin, sizeof(sin)) < 0) {
printf("err %d in connect with server, fd %u connect failed !\n", fd, errno);
continue ;
}

char buf[64] = {0};
sprintf(buf, "%d", i);
bufferevent_map.insert(make_pair(bufevt, 0));
bufferevent_write(bufevt, buf, strlen(buf));
usleep(10);
counter++;
}

printf("now EST connect count %d\n", counter);
write_count(counter);

evtimer_add(timeout, &tv);
event_base_dispatch(evbase);
evtimer_del(timeout);
event_base_free(evbase);

return 0;
}