lua 异步HTTPS并发请求库

项目使用skynet框架，这个框架主要用lua写逻辑，但缺乏对HTTPS支持，所以我利用一点时间写了lua模块，支持异步HTTPS请求，文章这里讲述HTTPS相关知识，如何接入openssl请求HTTPS数据，同时也分享了lua模块给大家参考。

HTTPS说明

HTTPS可以理解成 HTTP协议的安全版，协议还是HTTP协议，只是对传输过程的数据进行了加密处理，保证数据传输的安全。（默认端口是443）HTTPS验证数据的过程如下：

skynet HTTPS支持

skynet只封装了HTTP的接口，没有对HTTPS做支持，所以要外接lua 库使用。
skynet支持HTTPS请求有两种方法：

	优点	缺点
CURL	支持http/https/ftp等，接入较简单	并发支持差
OpenSSL	更底层，效率较高	接入复杂

其实，CURL是利用OpenSSL实现的，有网友封装了非阻塞版本的lua CURL库，可用于skynet处理HTTPS请求。链接猛击这里。
前面提到了CURL的缺点，CURL本身可做并发请求（libcurl multi），但做法却是将所有URL请求合到一起处理，需等全部URL数据处理完毕才返回数据。假设其中一个URL出现超时，那一起的其他URL都会受影响。
所以，文章推荐使用OpenSSL，这里先介绍C/C++如何处理HTTPS请求，然后再封装一个lua库，给大家演示下 skynet 如何请求HTTPS数据。

C/C++处理HTTPS请求

这里以一个例子，说明C/C++如何处理HTTPS请求。

#include <stdio.h>
#include <string.h>
#include <errno.h>
#include <sys/socket.h>
#include <resolv.h>
#include <stdlib.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include "openssl/ssl.h"
#include "openssl/err.h"

#define MAXBUF 1024

int main(int argc, char **argv)
{
int sockfd, len;
char sendFN[1024];
struct sockaddr_in dest;
char buffer[MAXBUF + 1];
SSL_CTX *ctx;
SSL *ssl;
char host_file[] = "";
char  host_addr[] = "www.jd.com";
char ip[] = "112.91.125.129";
int port = "443";

/* SSL 库初始化 */
SSL_library_init();
OpenSSL_add_all_algorithms();
SSL_load_error_strings();
ctx = SSL_CTX_new(SSLv23_client_method());
if (ctx == NULL)
{
ERR_print_errors_fp(stdout);
exit(1);
}
if ((sockfd = socket(AF_INET, SOCK_STREAM, 0)) < 0)
{
perror("Socket Create Fail!");
exit(errno);
}
/* 建立 TCP 连接 */
bzero(&dest, sizeof(dest));
dest.sin_family = AF_INET;
dest.sin_port = htons(atoi(port));
if (inet_aton(ip, (struct in_addr *) &dest.sin_addr.s_addr) == 0)
{
perror("Socket Init Fail!");
exit(errno);
}
printf("Socket Created\n");

if (connect(sockfd, (struct sockaddr *) &dest, sizeof(dest)) != 0)
{
perror("Socket Connect Fail!");
exit(errno);
}
printf("Socket Connected\n");

/* 绑定 Socket 与 SSL */
ssl = SSL_new(ctx);
SSL_set_fd(ssl, sockfd);
/* 建立 SSL 连接 */
if (SSL_connect(ssl) == -1)
ERR_print_errors_fp(stderr);
else
printf("SSL Connected with %s encryption\n", SSL_get_cipher(ssl));

sprintf(sendFN, "GET /%s HTTP/1.1\r\nHost: %s\r\nConnection: Close\r\n\r\n", host_file, host_addr);

/* SSL 发数据 */
len = SSL_write(ssl, sendFN, strlen(sendFN));
if (len < 0)
printf("SSL Send failure! errno = %d, err_msg = %s\n", errno, strerror(errno));

printf("SSL Send Done !\n\n");

bzero(buffer, MAXBUF + 1);
int nbytes;
/* SSL 收数据 */
while ((nbytes = SSL_read(ssl, buffer, 1)) == 1) {
/* 打印收到的数据 */
printf("%s", buffer);
}

printf("\n\nSSL Read Done !\n");

/* 关闭连接 */
SSL_shutdown(ssl);
SSL_free(ssl);
close(sockfd);
SSL_CTX_free(ctx);
return 0;
}

原理很简单，底层还是走了socket，只是TCP连接建立后，把连接交给SSL，让SSL收发数据。

Lua处理HTTPS请求

这里提供lua版本处理HTTPS请求，除了skynet，其他lua项目也可以使用。以下是lua C模块的代码，保存为lua_httpsc.c

#include <stdlib.h>
#include <string.h>
#include <lua.h>
#include <lauxlib.h>
#include <time.h>

#include <netinet/in.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <errno.h>
#include <fcntl.h>
#include "openssl/ssl.h"
#include "openssl/err.h"
#include <poll.h>

#define CACHE_SIZE 0x1000
#define ERROR_FD -1
#define SEND_RETRY 10

static SSL_CTX *ctx = NULL;

typedef struct {
int is_init;
} cutil_conf_t;

enum conn_st
{
CONNECT_INIT = 1,
CONNECT_PORT = 2,
CONNECT_SSL = 3,
CONNECT_DONE = 4
};

typedef struct {
int fd;
SSL* ssl;
enum conn_st status;
} cutil_fd_t;

static cutil_conf_t* fetch_config(lua_State *L) {
cutil_conf_t* cfg;
cfg = lua_touserdata(L, lua_upvalueindex(1));
if (!cfg)
luaL_error(L, "httpsc: Unable to fetch cfg");

return cfg;
}

static void close_fd_t(cutil_fd_t* fd_t) {
if ( fd_t == NULL )
return;

SSL* ssl = fd_t->ssl;
if ( ssl != NULL ) {
SSL_shutdown(ssl);
SSL_free(ssl);
}

int fd = fd_t->fd;
if (fd != ERROR_FD)
close(fd);

free(fd_t);
}

static int try_connect_ssl(SSL* ssl) {
int ret,err;
ret = SSL_connect(ssl);
err = SSL_get_error(ssl, ret);
if (ret == 1)
return 0;

if (err != SSL_ERROR_WANT_WRITE && err != SSL_ERROR_WANT_READ ) {
return -1;
}
return 1;
}

static int lconnect(lua_State *L) {
cutil_conf_t* cfg = fetch_config(L);
if(!cfg->is_init)
{
luaL_error(L, "httpsc: Not inited");
return 0;
}

const char * addr = luaL_checkstring(L, 1);
int port = luaL_checkinteger(L, 2);

cutil_fd_t* fd_t = (cutil_fd_t *)malloc(sizeof(cutil_fd_t));
if ( fd_t == NULL )
return luaL_error(L, "httpsc: Create fd %s %d failed", addr, port);
fd_t->fd = ERROR_FD;
fd_t->ssl = NULL;
fd_t->status = CONNECT_INIT;

int fd = socket(AF_INET, SOCK_STREAM, 0);
struct sockaddr_in my_addr;
fd_t->fd = fd;

bzero(&my_addr, sizeof(my_addr));
my_addr.sin_addr.s_addr = inet_addr(addr);
my_addr.sin_family = AF_INET;
my_addr.sin_port = htons(port);

int ret;
struct timeval timeo = {3, 0};
socklen_t len = sizeof(timeo);
ret = setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &timeo, len);
if (ret) {
close_fd_t(fd_t);
return luaL_error(L, "httpsc: Setsockopt %s %d failed", addr, port);
}

int flag = fcntl(fd, F_GETFL, 0);
fcntl(fd, F_SETFL, flag | O_NONBLOCK);

ret = connect(fd, (struct sockaddr *)&my_addr, sizeof(struct sockaddr_in));
if (ret != 0) {
if (errno == EINPROGRESS) {
fd_t->status = CONNECT_PORT;
} else {
close_fd_t(fd_t);
return luaL_error(L, "httpsc: Connect %s %d failed", addr, port);
}

} else {
fd_t->status = CONNECT_SSL;
SSL *ssl = SSL_new(ctx);
fd_t->ssl = ssl;
SSL_set_fd(ssl, fd);
ret = try_connect_ssl(ssl);
if (ret == 0) {
fd_t->status = CONNECT_DONE;
} else if (ret == -1) {
close_fd_t(fd_t);
return luaL_error(L, "httpsc ssl_connect error, errno = %d", errno);
}
}

lua_pushlightuserdata(L, fd_t);

return 1;
}

static int lcheck_connect(lua_State *L) {
cutil_fd_t* fd_t = (cutil_fd_t* ) lua_touserdata(L, 1);
if ( fd_t == NULL )
return luaL_error(L, "httpsc fd error");

switch (fd_t->status) {
case CONNECT_DONE:
lua_pushboolean(L, 1);
return 1;
case CONNECT_PORT:
{
struct pollfd fds;
int ret, err;
fds.fd = fd_t->fd;
fds.events = POLLIN | POLLOUT;
/* get status immediately */
ret = poll(&fds, 1, 0);

if (ret != -1) {
socklen_t len = sizeof(int);
ret = getsockopt(fd_t->fd, SOL_SOCKET, SO_ERROR, &err, &len);
if (ret < 0) {
close_fd_t(fd_t);
return luaL_error(L, "httpsc getsockopt error, ret = %d", ret);
}
if (err == 0) {
fd_t->status = CONNECT_SSL;
SSL *ssl = SSL_new(ctx);
fd_t->ssl = ssl;
SSL_set_fd(ssl, fd_t->fd);
ret = try_connect_ssl(ssl);
if (ret == 0) {
fd_t->status = CONNECT_DONE;
lua_pushboolean(L, 1);
return 1;
} else if (ret == -1) {
close_fd_t(fd_t);
return luaL_error(L, "httpsc connect ssl error, errno = %d", errno);
}
} else {
if (errno == EAGAIN || errno == EINTR || errno == EINPROGRESS ) {
return 0;
} else {
close_fd_t(fd_t);
return luaL_error(L, "httpsc connect sockopt error, errno = %d", errno);
}
}
} else {
close_fd_t(fd_t);
return luaL_error(L, "httpsc connect poll error, ret = %d", ret);
}
return 0;
}
case CONNECT_SSL:
{
int ret = try_connect_ssl(fd_t->ssl);
if (ret == 0) {
fd_t->status = CONNECT_DONE;
lua_pushboolean(L, 1);
return 1;
} else if (ret == -1) {
close_fd_t(fd_t);
return luaL_error(L, "httpsc connect ssl 2 error, errno = %d", errno);
}
return 0;
}
default:
;
}

close_fd_t(fd_t);
return luaL_error(L, "httpsc connect fator error");
}

static int lclose(lua_State *L) {
cutil_fd_t* fd_t = (cutil_fd_t* ) lua_touserdata(L, 1);
if ( fd_t == NULL )
return luaL_error(L, "httpsc fd error");

close_fd_t(fd_t);

return 0;
}

static void block_send(lua_State *L, SSL *ssl, const char * buffer, int sz) {
int retry = SEND_RETRY;
while(sz > 0) {
if (retry <= 0) {
luaL_error(L, "httpsc: socket error: retry timeout");
return;
}
retry -= 1;
int r = SSL_write(ssl, buffer, sz);
if (r < 0) {
if (errno == EAGAIN || errno == EINTR)
continue;
luaL_error(L, "httpsc: socket error: %s", strerror(errno));
return;
}
buffer += r;
sz -= r;
}
}

static int lsend(lua_State *L) {
cutil_conf_t* cfg = fetch_config(L);
if(!cfg->is_init)
{
luaL_error(L, "httpsc: Not inited");
return 0;
}

size_t sz = 0;
cutil_fd_t* fd_t = (cutil_fd_t* ) lua_touserdata(L, 1);
if ( fd_t == NULL )
return luaL_error(L, "httpsc fd error");
SSL* ssl = fd_t->ssl;
const char * msg = luaL_checklstring(L, 2, &sz);

block_send(L, ssl, msg, (int)sz);

return 0;
}

static int lrecv(lua_State *L) {
cutil_conf_t* cfg = fetch_config(L);
if(!cfg->is_init)
{
luaL_error(L, "httpsc: Not inited");
return 0;
}
cutil_fd_t* fd_t = (cutil_fd_t* ) lua_touserdata(L, 1);
if ( fd_t == NULL )
return luaL_error(L, "httpsc fd error");
SSL* ssl = fd_t->ssl;
int top = lua_gettop(L);

char buffer[CACHE_SIZE];
int size = CACHE_SIZE;
if ( top > 1 && lua_isnumber(L, 2)) {
int _size = lua_tointeger(L, 2);
size = _size > size ? size : _size;
}

int r = SSL_read(ssl, buffer, size);
// if (r == 0) {
// 	lua_pushliteral(L, "");
// 	/* close */
// 	return 1;
// }
if (r <= 0) {
if (errno == EAGAIN || errno == EINTR) {
return 0;
}
return luaL_error(L, "httpsc: socket error: %s", strerror(errno));
}
lua_pushlstring(L, buffer, r);
return 1;
}

static int lusleep(lua_State *L) {
int n = luaL_checknumber(L, 1);
usleep(n);
return 0;
}

/* GC, clean up the buf */
static int _gc(lua_State *L)
{
cutil_conf_t *cfg;
cfg = lua_touserdata(L, 1);

if (ctx != NULL){
SSL_CTX_free(ctx);
ctx = NULL;
}

/* todo: auto gc */

cfg = NULL;
return 0;
}

static void _create_config(lua_State *L)
{
cutil_conf_t *cfg;
cfg = lua_newuserdata(L, sizeof(*cfg));
cfg->is_init = !!NULL;
/* Create GC method to clean up buf */
lua_newtable(L);
lua_pushcfunction(L, _gc);
lua_setfield(L, -2, "__gc");
lua_setmetatable(L, -2);

/* openssl init */
if ( ctx == NULL) {
SSL_library_init();
OpenSSL_add_all_algorithms();
SSL_load_error_strings();
ctx = SSL_CTX_new(SSLv23_client_method());
if (ctx == NULL)
{
ERR_print_errors_fp(stdout);
luaL_error(L, "httpsc: Unable to init openssl");
return;
}
}

cfg->is_init = !NULL;
}

int luaopen_httpsc(lua_State *L)
{
static const luaL_Reg funcs[] = {
{ "connect", lconnect },
{ "check_connect", lcheck_connect },
{ "recv", lrecv },
{ "send", lsend },
{ "close", lclose },
{ "usleep", lusleep },
{NULL, NULL}
};

lua_newtable(L);
_create_config(L);
luaL_setfuncs(L, funcs, 1);

return 1;
}

编译到项目执行后，lua调用的方法如下：

local httpsc = require "httpsc"
for k,v in pairs(httpsc ) do
print(k,v)
end

local fd = httpsc.connect("163.177.151.109", 443)
print(fd)

while true do
local ok = httpsc.check_connect(fd)
if ok then break end
httpsc.usleep(10000)
end

-- httpsc.usleep(10000)
httpsc.send(fd, "GET / HTTP/1.1\r\nAccept: */*\r\nHost: www.baidu.com\r\nConnection: Close\r\n\r\n")

httpsc.usleep(1000000)

local body = ""
while true do
local r = httpsc.recv(fd)
print(r)
if r and #r>0 then
body = body .. r
else
break
end
end

print(body)

httpsc.close(fd)

print("ok!")

以上只是一个简单的例子，我同时也参考skynet clientsocket写了一个对HTTP协议支持友好的版本，代码托管在Git : https://github.com/chenweiqi/lua_httpsc ，欢迎交流。这个lua代码目前有个不足，没有做openssl多线程支持，不支持同时多个线程加载和调用模块，这个以后的版本会做优化

OpenSSL动态库的编译

既然基于OpenSSL开发，就需要编译OpenSSL库来使用。这里提供最近版本几个依赖库的编译方法：openssl  - libssl.a / libcrypto.a

wget https://www.openssl.org/source/openssl-1.0.2k.tar.gz tar -zxf openssl-1.0.2k.tar.gz
cd  openssl-1.0.2k
./config -fPIC enable-shared
make depend
make
make install
find -name "*.a"

zlib - zlib.a

wget http://zlib.net/zlib-1.2.11.tar.xz tar -zxf zlib-1.2.11.tar.xz
cd zlib-1.2.11
export CFLAGS=" -fPIC"
./configure
make
find -name "*.a"

idn - libidn.a

wget http://ftp.gnu.org/gnu/libidn/libidn-1.33.tar.gz tar -zxf libidn-1.33.tar.gz
cd libidn-1.33
export CFLAGS=" -fPIC"
./configure
make
find -name "*.a"

更新说明：2017/4/15 优化socket创建连接过程，由阻塞改成非阻塞方式2017/6/9 标题由“skynet lua 支持HTTPS请求”改成“lua 异步HTTPS并发请求库”
参考：http://blog.csdn.net/mycwq/article/details/64440253