docker 1.12 网络和负载均衡初探
2016-11-01 16:03
411 查看
主机A euca- 10-153-177-58
主机B euca-10-153-177-76
root@euca-10-153-177-58:~# dockernode list
ID HOSTNAME STATUS AVAILABILITY MANAGER STATUS
3ddwxfmfmpvkndj0l4ynzhnhr euca-10-153-177-76 Ready Active
cvfd6ufl2kwflqsga7df7rcl1 * euca-10-153-177-58 Ready Active Leader
root@euca-10-153-177-58:~# dockerservice list
ID NAME REPLICAS IMAGE COMMAND
aay34mgae9zf cloudbts 2/2 cloudbts:v4.0
d8mt79by1609 cbpts70 2/2 cbpts70:v3.0
service cloudbts 开放8180:8080 8122:22 端口映射
service cbpts70 开放8080:8080 8022:22 端口映射
查看NAT
root@euca-10-153-177-58:~# iptables-t nat –S
-A POSTROUTING -s 172.18.0.0/16 ! -odocker_gwbridge -j MASQUERADE
-A DOCKER -i docker0 -j RETURN
-A DOCKER -i docker_gwbridge -jRETURN
-A DOCKER-INGRESS -p tcp -m tcp--dport 8022 -j DNAT --to-destination 172.18.0.2:8022
-A DOCKER-INGRESS -p tcp -m tcp--dport 8080 -j DNAT --to-destination 172.18.0.2:8080
-A DOCKER-INGRESS -p tcp -m tcp--dport 8180 -j DNAT --to-destination 172.18.0.2:8180
-A DOCKER-INGRESS -p tcp -m tcp--dport 8122 -j DNAT --to-destination 172.18.0.2:8122
root@euca-10-153-177-76:~# iptables-t nat –S
-A POSTROUTING -s 172.17.0.0/16 ! -odocker0 -j MASQUERADE
-A POSTROUTING -o docker_gwbridge -maddrtype --src-type LOCAL -j MASQUERADE
-A POSTROUTING -s 172.18.0.0/16 ! -odocker_gwbridge -j MASQUERADE
-A DOCKER -i docker0 -j RETURN
-A DOCKER -i docker_gwbridge -jRETURN
-A DOCKER-INGRESS -p tcp -m tcp--dport 8022 -j DNAT --to-destination 172.18.0.2:8022
-A DOCKER-INGRESS -p tcp -m tcp --dport8080 -j DNAT --to-destination 172.18.0.2:8080
-A DOCKER-INGRESS -p tcp -m tcp--dport 8180 -j DNAT --to-destination 172.18.0.2:8180
-A DOCKER-INGRESS -p tcp -m tcp--dport 8122 -j DNAT --to-destination 172.18.0.2:8122
检查docker 网络命名空间列表
root@euca-10-153-177-58:~# ls/var/run/docker/netns
1-22hxmh4c0e caf29e8b2fab d596a954d729 ingress_sbox
root@euca-10-153-177-76:~# ls/var/run/docker/netns
1-22hxmh4c0e c329f7db767d f08c5ce5be91 ingress_sbox
进入ingress_sbox网络命名空间
root@euca-10-153-177-76:/var/run/docker/netns#nsenter --net=ingress_sbox sh
# ifconfig
eth0 Link encap:Ethernet HWaddr 02:42:0a:ff:00:07
inet addr:10.255.0.7 Bcast:0.0.0.0 Mask:255.255.0.0
inet6 addr: fe80::42:aff:feff:7/64 Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1450 Metric:1
RX packets:112920 errors:0 dropped:0overruns:0 frame:0
TX packets:105006 errors:0 dropped:0overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:414828822 (414.8 MB) TX bytes:8263325 (8.2 MB)
eth1 Link encap:Ethernet HWaddr 02:42:ac:12:00:02
inet addr:172.18.0.2 Bcast:0.0.0.0 Mask:255.255.0.0
inet6 addr: fe80::42:acff:fe12:2/64Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:105086 errors:0 dropped:0overruns:0 frame:0
TX packets:112895 errors:0 dropped:0overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:8268245 (8.2 MB) TX bytes:414825888 (414.8 MB)
lo Link encap:Local Loopback
inet addr:127.0.0.1 Mask:255.0.0.0
inet6 addr: ::1/128 Scope:Host
UP LOOPBACK RUNNING MTU:65536 Metric:1
RX packets:0 errors:0 dropped:0 overruns:0frame:0
TX packets:0 errors:0 dropped:0overruns:0 carrier:0
collisions:0 txqueuelen:1
RX bytes:0 (0.0 B) TX bytes:0 (0.0 B)
# iptables -nvL -t mangle
Chain PREROUTING (policy ACCEPT 229Kpackets, 442M bytes)
pkts bytes target prot opt in out source destination
0 0 MARK tcp -- * * 0.0.0.0/0 0.0.0.0/0 tcp dpt:8122MARK set 0x103
181 12004 MARK tcp -- * * 0.0.0.0/0 0.0.0.0/0 tcp dpt:8180 MARK set 0x103
110K 7153K MARK tcp -- * * 0.0.0.0/0 0.0.0.0/0 tcp dpt:8080MARK set 0x104
0 0 MARK tcp -- * * 0.0.0.0/0 0.0.0.0/0 tcp dpt:8022 MARK set 0x104
Chain INPUT (policy ACCEPT 110Kpackets, 7161K bytes)
pkts bytes target prot opt in out source destination
Chain FORWARD (policy ACCEPT 119Kpackets, 435M bytes)
pkts bytes target prot opt in out source destination
Chain OUTPUT (policy ACCEPT 110Kpackets, 7161K bytes)
pkts bytes target prot opt in out source destination
0 0 MARK all -- * * 0.0.0.0/0 10.255.0.2 MARK set0x103
0 0 MARK all -- * * 0.0.0.0/0 10.255.0.6 MARK set0x104
Chain POSTROUTING (policy ACCEPT229K packets, 442M bytes)
pkts bytes target prot opt in out source destination
# ipvsadm
IP Virtual Server version 1.2.1(size=4096)
Prot LocalAddress:Port SchedulerFlags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
FWM 259 (=x103)rr
-> 10.255.0.4:0 Masq 1 0 0
-> 10.255.0.5:0 Masq 1 0 0
FWM 260(=0x104)rr
-> 10.255.0.8:0 Masq 1 0 9
-> 10.255.0.9:0
f0b1
Masq 1 0 9
进入1-22hxmh4c0e 网络命名空间
root@euca-10-153-177-76:/var/run/docker/netns#nsenter --net=1-22hxmh4c0e sh
# ifconfig
br0 Link encap:Ethernet HWaddr 36:ca:0a:78:14:ae
inet addr:10.255.0.1 Bcast:0.0.0.0 Mask:255.255.0.0
inet6 addr: fe80::b45f:c9ff:fe6c:7216/64Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1450 Metric:1
RX packets:55 errors:0 dropped:0overruns:0 frame:0
TX packets:8 errors:0 dropped:0overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:3412 (3.4 KB) TX bytes:648 (648.0 B)
lo Link encap:Local Loopback
inet addr:127.0.0.1 Mask:255.0.0.0
inet6 addr: ::1/128 Scope:Host
UP LOOPBACK RUNNING MTU:65536 Metric:1
RX packets:0 errors:0 dropped:0overruns:0 frame:0
TX packets:0 errors:0 dropped:0overruns:0 carrier:0
collisions:0 txqueuelen:1
RX bytes:0 (0.0 B) TX bytes:0 (0.0 B)
veth2 Link encap:Ethernet HWaddr 36:ca:0a:78:14:ae
inet6 addr:fe80::34ca:aff:fe78:14ae/64 Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1450 Metric:1
RX packets:114691 errors:0 dropped:0overruns:0 frame:0
TX packets:123664 errors:0 dropped:0overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:9026640 (9.0 MB) TX bytes:449408513 (449.4 MB)
veth6 Link encap:Ethernet HWaddr d6:2e:06:03:e9:7e
inet6 addr:fe80::d42e:6ff:fe03:e97e/64 Scope:Link
UP BROADCAST RUNNINGMULTICAST MTU:1450 Metric:1
RX packets:446 errors:0 dropped:0overruns:0 frame:0
TX packets:428 errors:0 dropped:0overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:860524 (860.5 KB) TX bytes:42790 (42.7 KB)
veth7 Link encap:Ethernet HWaddr 42:63:b9:1e:1f:78
inet6 addr:fe80::4063:b9ff:fe1e:1f78/64 Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1450 Metric:1
RX packets:86968 errors:0 dropped:0overruns:0 frame:0
TX packets:58728 errors:0 dropped:0overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:237555101 (237.5 MB) TX bytes:4590982 (4.5 MB)
vxlan1 Link encap:Ethernet HWaddr 5e:90:7c:34:53:d9
inet6 addr:fe80::5c90:7cff:fe34:53d9/64 Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1450 Metric:1
RX packets:158534 errors:0 dropped:0overruns:0 frame:0
TX packets:57099 errors:0 dropped:59overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:218690170 (218.6 MB) TX bytes:5708548 (5.7 MB)
# ip -d link show vxlan1
12: vxlan1:<BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue master br0 stateUNKNOWN mode DEFAULT group default
link/ether 5e:90:7c:34:53:d9 brd ff:ff:ff:ff:ff:ff link-netnsid 0promiscuity 1
vxlan id 256 srcport 0 0 dstport 4789 proxy l2miss l3miss ageing 300
bridge_slavestate forwarding priority 32 cost 100 hairpin off guard off root_block offfastleave off learning on flood on addrgenmode eui64
root@euca-10-153-177-58:/var/run/docker/netns#nsenter --net=1-22hxmh4c0e sh
#ifconfig
br0 Link encap:Ethernet HWaddr 22:d7:ae:ab:99:36
inet addr:10.255.0.1 Bcast:0.0.0.0 Mask:255.255.0.0
inet6 addr:fe80::ac88:34ff:feb1:f213/64 Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1450 Metric:1
RX packets:48 errors:0 dropped:0overruns:0 frame:0
TX packets:8 errors:0 dropped:0 overruns:0carrier:0
collisions:0 txqueuelen:0
RX bytes:2904 (2.9 KB) TX bytes:648 (648.0 B)
lo Link encap:Local Loopback
inet addr:127.0.0.1 Mask:255.0.0.0
inet6 addr: ::1/128 Scope:Host
UP LOOPBACK RUNNING MTU:65536 Metric:1
RX packets:0 errors:0 dropped:0overruns:0 frame:0
TX packets:0 errors:0 dropped:0overruns:0 carrier:0
collisions:0 txqueuelen:1
RX bytes:0 (0.0 B) TX bytes:0 (0.0 B)
veth2 Link encap:Ethernet HWaddr e6:f7:58:70:40:61
inet6 addr:fe80::e4f7:58ff:fe70:4061/64 Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1450 Metric:1
RX packets:4489 errors:0 dropped:0overruns:0 frame:0
TX packets:2850 errors:0 dropped:0overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:316738 (316.7 KB) TX bytes:10033175 (10.0 MB)
veth5 Link encap:Ethernet HWaddr 2a:9e:6c:9c:30:99
inet6 addr:fe80::289e:6cff:fe9c:3099/64 Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1450 Metric:1
RX packets:442 errors:0 dropped:0overruns:0 frame:0
TX packets:488 errors:0 dropped:0overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:761515 (761.5 KB) TX bytes:46143 (46.1 KB)
veth6 Link encap:Ethernet HWaddr 22:d7:ae:ab:99:36
inet6 addr:fe80::20d7:aeff:feab:9936/64 Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1450 Metric:1
RX packets:121565 errors:0 dropped:0overruns:0 frame:0
TX packets:81680 errors:0 dropped:0overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:325510290 (325.5 MB) TX bytes:6450337 (6.4 MB)
vxlan1 Link encap:Ethernet HWaddr 86:4f:c5:4d:12:18
inet6 addr:fe80::844f:c5ff:fe4d:1218/64 Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1450 Metric:1
RX packets:82661 errors:0 dropped:0overruns:0 frame:0
TX packets:122168 errors:0 dropped:55overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:7356251 (7.3 MB) TX bytes:322463758 (322.4 MB)
# ip -dlink show vxlan1
6:vxlan1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue masterbr0 state UNKNOWN mode DEFAULT group default
link/ether 86:4f:c5:4d:12:18 brdff:ff:ff:ff:ff:ff link-netnsid 0 promiscuity 1
vxlan id 256 srcport 0 0 dstport 4789 proxyl2miss l3miss ageing 300
bridge_slave state forwarding priority 32cost 100 hairpin off guard off root_block off fastleave off learning on floodon addrgenmode eui64
通过上述查看可见内部网络拓扑
如果外部访问服务,docker负载均衡的流程如下
1、 用户访问xx.xx.xx.76的8080端口。
2、 Iptable DNAT转发至 Ingress NS 空间里的172.18.0.2。
Ingress NS 空间里的Iptable根据不同的端口设置不同的Mark。
而服务IP 10.255.0.2是VIP,在10.255.0.2和10.255.0.4之间浮动。
FWM 259 (=x103)rr
->10.255.0.4:0 Masq 1 0 0
->10.255.0.5:0 Masq 1 0 0
3、 跳转至10.255.0.5,如果该IP在本容器,根据路由,数据包进入1-22xxx NS空间,
3.1然后通过veth至容器命名空间
4、 跳转至10.255.0.4,如果该IP不在本容器,根据路由,数据包进入1-22xxx NS空间。
5、 在1-22xxx NS空间里br0网桥进入vxlan link,
6、 vxlan link 最终进入物理网卡
7、 物理网卡传输至远程主机
8、 进入远程1-22xxx NS空间,并且vxlan 解包
9、 跳转至容器空间。
如果是容器内部主动往外发送数据,流程如下图:
容器访问网关docker_gwbridge,由docker_gwbridge SNAT转发至外部网络。
总的来说,docker 使用VIP来实现了负载均衡,相比较k8s使用kube-proxy 管理iptable来实现的负载均衡,看起来效率高点,但还是避免不了iptable 转发对网络效率的影响。
而且,docker 的service 对外暴露可访问IP是一刀切的,所有节点都暴露,管理粗细度还是不及k8s,k8s可单个节点暴露。
相关文章推荐
- Docker1.12版本swarm模式下的网络模型
- Docker1.12版本swarm模式下的网络模型
- DockOne微信分享(六十六): Docker网络方案初探
- (转) Docker - Docker1.12服务发现,负载均衡和Routing Mesh
- 利用ansible centos7 系统批量安装docker1.12 并添加docker macvlan 网络支持
- Docker网络方案初探
- Docker网络方案初探
- apache2.2.4 负载均衡初探
- IT系统运维初探――中石化齐鲁分公司储运厂网络管理
- 初探基于CDMA网络的移动数据通信系统
- PVLAN技术初探-巧用PVLAN优化网络
- 在Ruby中进行网络协议开发初探
- 蛙蛙推荐:三层网络架构和负载均衡基础知识整理
- VRRP技术实现网络的路由冗余和负载均衡
- 使用网络地址转换实现多服务器负载均衡