利用iptabls的NFLOG记录自己的HTTP HTTPS上网行为
2017-07-20 09:07
911 查看
利用iptabls的NFLOG记录自己的HTTP HTTPS上网行为
想要记录上网行为(一天都访问了什么网站)势必要解析TCP数据包,第一种方法是自己写一个内核模块来做这件事,但调内核模块一崩溃系统就要重启,自己的真机上不方便调试。第二种方法是借助iptables的日志功能把数据包发到应用层来解析。iptables有几个日志记录的target(LOG, ULOG, NFLOG), LOG只是简单的记录包头信息到系统日志,而ULOG, NFLOG则可以把整个数据包发到应用层,应用层可以通过解析数据包来记录自己想要的信息。
大致的过程是这样:内核数据包走到iptables的钩子点上,如果数据包和规则匹配,就通过netlink把这个数据包多播到多播组,然后应用层通过netlink来接受这个数据包。
可以通过
iptables -j ULOG --help查看帮助信息,但自己添加iptables规则时(如:
sudo iptables -A OUTPUT -j ULOG)发现提示:
iptables: No chain/target/match by that name.
使用命令
cat /boot/config-$(uname -r) | grep ULOG得知内核没有编译ULOG进去,但是NFLOG是支持的。于是下面的程序使用NFLOG实现。
数据记录到文件,以后需要时可以再做分析:
1.tail /home/website_access_stat.txt 2.p1.music.126.net, 2017-07-19 22:17, 1500473873 3.p1.music.126.net, 2017-07-19 22:17, 1500473873 4.highlightjs.org, 2017-07-19 22:17, 1500473875 5.music.163.com, 2017-07-19 22:17, 1500473876 6.music.163.com, 2017-07-19 22:17, 1500473877 7.music.163.com, 2017-07-19 22:17, 1500473877 8.music.163.com, 2017-07-19 22:17, 1500473877 9.m8.music.126.net, 2017-07-19 22:17, 1500473877 10.music.163.com, 2017-07-19 22:17, 1500473877 11.www.gstatic.com, 2017-07-19 22:17, 1500473877
实现的代码:
1./************************************************************ 2.Copyright (C), 2017, Leon, All Rights Reserved. 3.FileName: http_trace.c 4.Description: 跟踪自己的http,https请求 5.Author: Leon 6.Version: 1.0 7.Date: 2017-7-19 13:47:04 8.Function: 9. 10.History: 11.<author> <time> <version> <description> 12. Leon 13. 14. 安装依赖库: 15. apt install libnetfilter-log-dev libnfnetlink-dev 16. 17. 编译方法: 18. gcc http_trace.c -lnetfilter_log 19. 20. 需要以sudo权限运行 21. sudo iptables -I OUTPUT -o enp8s0 -p tcp -j NFLOG --nflog-group 1 22. sudo ./a.out & 23. 24. 统计方法: 25. cat /home/website_access_stat.txt | awk -F "," '{stat[$1]++} END {for(cmd in stat) print stat[cmd] " " cmd}' | sort -nr 26. ************************************************************/ 27. 28.#include <stdio.h> 29.#include <stdlib.h> 30.#include <string.h> 31.#include <sys/socket.h> 32.#include <arpa/inet.h> 33.#include <sys/types.h> 34.#include <linux/netlink.h> 35.#include <sys/select.h> 36.#include <sys/time.h> 37.#include <unistd.h> 38.#include <time.h> 39. 40.#include <libnetfilter_log/libnetfilter_log.h> 41. 42.#include <linux/ip.h> 43.#include <linux/tcp.h> 44.#include <linux/in.h> 45. 46.#include "https.h" 47. 48.#define LISTEN_GROUP 1 49.#define BUFF_SIZE 4096 50. 51.#define DATA_FILE "/home/website_access_stat.txt" 52. 53.#define lprintf(format, argv...) printf("%s(%d): " format , __FUNCTION__, __LINE__, ##argv) 54. 55.static char host_data[BUFF_SIZE] = {0}; 56.static int data_idx = 0; /* 指向host_data的当前存储位置 */ 57. 58.void print_pkt(unsigned char *pkt, int len, char *title) 59.{ 60. char hex[64] = {0}; 61. char assic[64] = {0}; 62. char *p_hex = hex, *p_assic = assic; 63. int i = 0; 64. 65. printf("*******************%s start*****************\n", title); 66. while(len--) 67. { 68. if(i != 0 && !(i%16)) 69. { 70. printf("%s %s\n", hex, assic); 71. memset(hex, 0x0, sizeof(hex)); 72. memset(assic, 0x0, sizeof(assic)); 73. p_hex = hex; 74. p_assic = assic; 75. i = 0; 76. } 77. sprintf(p_hex, "%02x ", *pkt); 78. p_hex += 3; 79. sprintf(p_assic, "%c", *pkt < 32 || *pkt > 127? '.' : *pkt); 80. p_assic += 1; 81. i++; 82. pkt++; 83. if(!(i%8)) 84. { 85. sprintf(p_hex, " "); 86. p_hex += 2; 87. sprintf(p_assic, " "); 88. p_assic += 2; 89. } 90. } 91. printf("%s %s\n", hex, assic); 92. printf("\n\n*******************%s end*****************\n", title); 93.} 94. 95.void save_to_file(void) 96.{ 97. int len = data_idx; 98. int write_len = 0; 99. 100. if(!data_idx) 101. return; 102. 103. FILE *fp = fopen(DATA_FILE, "a+"); 104. if(!fp) 105. { 106. perror("fopen"); 107. return; 108. } 109. write_len = fwrite(host_data, len, 1, fp); 110. 111. fclose(fp); 112. data_idx = 0; 113.} 114. 115.void save_server_name(char *server_name) 116.{ 117. time_t sec = time(0); 118. struct tm now; 119. char buf[1024] = {0}; 120. int ret = 0; 121. 122. localtime_r(&sec, &now); 123. now.tm_year += 1900; 124. now.tm_mon += 1; 125. 126. ret = snprintf(buf, sizeof(buf) - 1, "%s, %d-%02d-%02d %02d:%02d, %u\n", 127. server_name, now.tm_year, now.tm_mon, now.tm_mday, now.tm_hour, 128. now.tm_min, (unsigned int)sec); 129. 130. if(ret + data_idx > BUFF_SIZE) 131. { 132. save_to_file(); 133. } 134. 135. strcpy(host_data + data_idx, buf); 136. data_idx += ret; 137.} 138. 139.int find_server_name(__u8 *edata, int data_len, char *server_name, int sn_len) 140.{ 141. __u8 *p = edata; 142. __u16 type; 143. __u16 len; 144. 145. while(p < edata + data_len) 146. { 147. type = ntohs(*(__u16 *)p); 148. p += 2; 149. len = ntohs(*(__u16 *)p); 150. p += 2; 151. 152. if(type == SERVER_NAME) 153. { 154. p += 3; /* 跳过server name list length和server name type字段 */ 155. len = ntohs(*(__u16 *)p); 156. p += 2; 157. if(len > sn_len) 158. { 159. memcpy(server_name, p, sn_len); 160. server_name[sn_len - 1] = '\0'; 161. } 162. else 163. { 164. memcpy(server_name, p, len); 165. server_name[len] = '\0'; 166. } 167. return 1; 168. } 169. p += len; 170. } 171. 172. return 0; 173.} 174. 175. 176.#define POINT_OVERFLOW_CHECK(point, limit) do{\ 177. if((point) > (limit)) \ 178. goto point_overflow; \ 179. }while(0) 180. 181.void https_handle(struct iphdr *iph) 182.{ 183. struct tcphdr *tcph = (void*)iph + iph->ihl*4; 184. __u8 *tcpdata = (void*)tcph + tcph->doff*4; 185. int len = ntohs(iph->tot_len) - (iph->ihl*4) - (tcph->doff*4); 186. 187. struct https_head *https = (struct https_head *)tcpdata; 188. __u8 *p = tcpdata; 189. __u16 tmp_len; 190. __u8 *point_limit = tcpdata + len; 191. char server_name[HOSTNAME_MAX_LEN] = {0}; 192. 193. if(len < sizeof(struct https_head)) 194. return; 195. 196. if(https->content_type != HANDSHAKE) 197. return; 198. 199. if(https->handshake_type != CLIENT_HELLO) 200. return; 201. 202. /* TLS1.0开始支持server name字段 */ 203. if(https->version < htons(TLS1_0_VERSION) 204. && https->version2 < htons(TLS1_0_VERSION)) 205. return; 206. 207. p = &(https->session_id_length); 208. 209. /* 跳过session_id_length字段 */ 210. tmp_len = *p; 211. p++; 212. p += tmp_len; 213. POINT_OVERFLOW_CHECK(p, point_limit); 214. 215. /* 跳过cipher_suites_length字段 */ 216. tmp_len = ntohs(*(__u16 *)p); 217. p += 2; 218. p += tmp_len; 219. POINT_OVERFLOW_CHECK(p, point_limit); 220. 221. /* 跳过compression_methonds_length字段 */ 222. tmp_len = *p; 223. p++; 224. p += tmp_len; 225. POINT_OVERFLOW_CHECK(p, point_limit); 226. 227. /* 现在为扩展字段 */ 228. tmp_len = ntohs(*(__u16 *)p); 229. p += 2; 230. 231. if(0 == find_server_name(p, tmp_len, server_name, sizeof(server_name))) 232. { 233. return; 234. } 235. else 236. { 237. save_server_name(server_name); 238. return; 239. } 240. 241.point_overflow: 242. lprintf("point overflow\n"); 243. return; 244. 245.} 246. 247./* 不区分大小写的strstr */ 248.char *strncasestr(char *str, char *sub) 249.{ 250. if(!str || !sub) 251. return NULL; 252. 253. int len = strlen(sub); 254. if (len == 0) 255. { 256. return NULL; 257. } 258. 259. while (*str) 260. { 261. if (strncasecmp(str, sub, len) == 0) 262. { 263. return str; 264. } 265. ++str; 266. } 267. return NULL; 268.} 269. 270./* 从字符串缓冲区读取一行数据(跳过空行),返回下一个字符串的开始位置 */ 271.char *readline_from_buf(char *from, int from_len, char *to, int to_len) 272.{ 273. int i = 0; 274. 275. memset(to, 0x0, to_len); 276. 277. while(*from == '\r' || *from == '\n') 278. { 279. from++; 280. from_len--; 281. } 282. 283. for(i = 0; i < to_len && i < from_len; i++) 284. { 285. if(from[i] == '\r' || from[i] == '\n') 286. { 287. memcpy(to, from, i); 288. return from + i; 289. } 290. } 291. return NULL; 292.} 293. 294.void http_handle(struct iphdr *iph) 295.{ 296. struct tcphdr *tcph = (void*)iph + iph->ihl*4; 297. __u8 *tcpdata = (void*)tcph + tcph->doff*4; 298. int len = ntohs(iph->tot_len) - (iph->ihl*4) - (tcph->doff*4); 299. char line[1024]; 300. char *p = NULL; 301. 302. if(len && tcpdata[0] != 'G' && tcpdata[0] != 'P') 303. return; 304. 305. while(tcpdata = readline_from_buf(tcpdata, len, line, sizeof(line))) 306. { 307. //printf("%s\n", line); 308. if(strncasestr(line, "host:")) 309. { 310. p = (char *)line + strlen("host:"); 311. while(*p == ' ') 312. p++; 313. save_server_name(p); 314. break; 315. } 316. } 317.} 318. 319. 320.static int cb(struct nflog_g_handle *gh, struct nfgenmsg *nfmsg, 321. struct nflog_data *nfa, void *data) 322.{ 323. struct nfulnl_msg_packet_hdr *ph = nflog_get_msg_packet_hdr(nfa); 324. u_int32_t mark = nflog_get_nfmark(nfa); 325. u_int32_t indev = nflog_get_indev(nfa); 326. u_int32_t outdev = nflog_get_outdev(nfa); 327. char *prefix = nflog_get_prefix(nfa); 328. char *payload; 329. int payload_len = nflog_get_payload(nfa, &payload); 330. struct iphdr *iph = (struct iphdr *)payload; 331. 332. struct tcphdr *tcph; 333. 334. if (iph->protocol != IPPROTO_TCP) /* not TCP */ 335. return 0; 336. 337. tcph = (void*)iph + iph->ihl*4; 338. 339. if(tcph->dest == htons(443)) 340. { 341. https_handle(iph); 342. return 0; 343. } 344. 345. if(tcph->dest == htons(80)) 346. { 347. http_handle(iph); 348. return 0; 349. } 350. 351. return 0; 352.} 353. 354.int main(int argc, char *argv[]) 355.{ 356. char buf[BUFF_SIZE] = {0}; 357. int len = 0; 358. struct timeval tv = {10, 0}; 359. int ret = 0; 360. fd_set rfds; 361. struct nflog_handle *h; 362. struct nflog_g_handle *gh; 363. int fd; 364. 365. h = nflog_open(); 366. if (!h) { 367. fprintf(stderr, "error during nflog_open()\n"); 368. exit(1); 369. } 370. 371. if (nflog_bind_pf(h, AF_INET) < 0) { 372. fprintf(stderr, "error during nflog_bind_pf()\n"); 373. exit(1); 374. } 375. 376. gh = nflog_bind_group(h, 1); 377. if (!gh) { 378. fprintf(stderr, "no handle for grup 1\n"); 379. exit(1); 380. } 381. 382. if (nflog_set_mode(gh, NFULNL_COPY_PACKET, 0xffff) < 0) { 383. fprintf(stderr, "can't set packet copy mode\n"); 384. exit(1); 385. } 386. 387. fd = nflog_fd(h); 388. 389. nflog_callback_register(gh, &cb, NULL); 390. 391. while(1) 392. { 393. tv.tv_sec = 5; 394. tv.tv_usec = 0; 395. FD_ZERO(&rfds); 396. FD_SET(fd, &rfds); 397. 398. ret = select(fd + 1, &rfds, NULL, NULL, &tv); 399. if(ret > 0) 400. { 401. len = recv(fd, buf, sizeof(buf), 0); 402. if(len) 403. { 404. nflog_handle_packet(h, buf, len); 405. } 406. } 407. else if(ret == 0) 408. { 409. save_to_file(); 410. } 411. } 412. 413. return 0; 414.}
还可以通过sting模块直接匹配字符串,然后再传到应用层处理,这样可以避免传递太多的数据包到应用层
iptables -m string –help
一些有帮助的网页:
ip6tables中nflog的使用
github中保存位置:
https://github.com/leon0625/practice/tree/master/http_trace
相关文章推荐
- 自己动手利用Socket 实现HTTP与HTTPS
- 记录一次http切换成https,并修改域名过程的坑自己的经历
- squid代理http和https方式上网的操作记录
- 利用开源工具实现轻量级上网行为审计(来源ispublic.com)
- nginx配置http和https记录
- http和https80端口是为HTTP(HyperText Transport Protocol)即超文本传输协议开放的,此为上网冲浪使用次数最多的协议,主要用于WWW(World Wide Web
- 利用HttpModule做流量记录 画蛇添足最后一笔
- 利用HttpModule做流量记录 画蛇添足最后一笔
- 利用一切机会丰富自己的知识,利用一切机会调整自己的行为,为了达成目标而与他人合作,取得共赢 - update by June 2012
- 自己写的关于LINQ to sql类的利用反射的深复制,做个记录
- MAC系统利用charles抓取微信小程序和手机APP数据包(http和https数据包)
- 利用httpClient发起https请求
- windows上利用charles抓取微信小程序数据包,手机APP数据包(HTTP与HTTPS数据包)
- 利用HttpModule做流量记录 补充
- 记录一波自己的蠢蛋行为(1)
- HTTP/HTTPS自动加密上网方案
- 记录利用tomcat服务器配置https双向认证配置过程
- 监控自己APP的http/https网络请求的地址和请求耗时
- C#利用HttpWebRequest进行post请求的示例(HTTPS)