Cloud foundry warden 源码学习(4)

wshd是C写的一个程序，侦听在unix socket端口，wsh是其客户端。

int parent_run(wshd_t *w) {
char path[MAXPATHLEN];
int rv;
pid_t pid;

memset(path, 0, sizeof(path));

strcpy(path + strlen(path), w->run_path);
strcpy(path + strlen(path), "/");
strcpy(path + strlen(path), "wshd.sock");

w->fd = un_listen(path);

rv = barrier_open(&w->barrier_parent);
assert(rv == 0);

rv = barrier_open(&w->barrier_child);
assert(rv == 0);

/* Unshare mount namespace, so the before clone hook is free to mount
* whatever it needs without polluting the global mount namespace. */
rv = unshare(CLONE_NEWNS);
assert(rv == 0);

rv = run(w->lib_path, "hook-parent-before-clone.sh");
assert(rv == 0);

pid = child_start(w);
assert(pid > 0);

parent_setenv_pid(w, pid);

rv = run(w->lib_path, "hook-parent-after-clone.sh");
assert(rv == 0);

rv = barrier_signal(&w->barrier_parent);
if (rv == -1) {
fprintf(stderr, "Error waking up child process\n");
exit(1);
}

rv = barrier_wait(&w->barrier_child);
if (rv == -1) {
fprintf(stderr, "Error waiting for acknowledgement from child process\n");
exit(1);
}

return 0;
}

int main(int argc, char **argv) {
wshd_t *w;
int rv;

/* Continue child execution in the context of the container */
if (argc > 1 && strcmp(argv[1], "--continue") == 0) {
return child_continue(argc, argv);
}

w = calloc(1, sizeof(*w));
assert(w != NULL);

rv = wshd__getopt(w, argc, argv);
if (rv == -1) {
exit(1);
}

if (strlen(w->run_path) == 0) {
strcpy(w->run_path, "run");
}

if (strlen(w->lib_path) == 0) {
strcpy(w->lib_path, "lib");
}

if (strlen(w->root_path) == 0) {
strcpy(w->root_path, "root");
}

assert_directory(w->run_path);
assert_directory(w->lib_path);
assert_directory(w->root_path);

parent_run(w);

return 0;
}


Wshd进程首先启动在unix socket上的侦听，接下来调用hook script完成 warden虚拟机的创建.

Skeleton/lib/hook-parent-before-clone.sh

mount -n -t aufs -o br:tmp/rootfs=rw:$rootfs_path=ro+wh none<pre class="plain" name="code">cp bin/wshd mnt/sbin/wshd
chmod 700 mnt/sbin/wshd

Skeleton/lib/hook-parent-after-clone.sh

# Add new group for every subsystem
for system_path in /tmp/warden/cgroup/*
do
instance_path=$system_path/instance-$id

mkdir -p $instance_path

if [ $(basename $system_path) == "cpuset" ]
then
cat $system_path/cpuset.cpus > $instance_path/cpuset.cpus
cat $system_path/cpuset.mems > $instance_path/cpuset.mems
fi
done

ip link add name $network_host_iface type veth peer name $network_container_iface
ip link set $network_host_iface netns 1
ip link set $network_container_iface netns $PID

ifconfig $network_host_iface $network_host_ip netmask $network_netmask

注意以上只是初始化cgroup，并没有对cpu/mem/disk/network做资源限制，这些会由DEA继续发limit request来完成。

紧接着，DEA会发送几个limit request，对warden container的资源做进一步的配置，这其中包括do_limit_memory, do_limit_cpu, do_limit_quota, do_limit_bandwith.

内存限制: 通过往cgroup mem里面memory.limit_in_bytes和memory.memsw.limit_in_bytes写入值限制该warden container的内存；

CPU限制: 通过往cgroup cpu里面cpu.shares写值。

Disk 限制: 通过调用setquota –u UID block_sock=xxx block_hard=xxx inode_soft=xxx inode_hard=xxx, 而UID是warden小虚机的/home/vcap目录。这就是说配给warden小虚机的disk配额。



至此，warden container创建完成。

总结一下：

Warden::Container::Linux在接收到create请求以后，拷贝skeleton文件系统，开启wshd进程，将wshd进程加入control group(cpu/memory/disk)

Warden container linux文件系统构成：

1./var/vcap/data/warden/depot/17r47v9oif1/tmp/rootfs
(instance data以aufs方式readwrite挂载 home, etc,dev,sbin,var)
---->17vedc80gj3/mnt
2./var/vcap/data/packages/rootfs_lucid64/2.1 (rootfs 以aufs方式readonly挂载)
---->17vedc80gj3/mnt
3.var/vcap/packages/buildpack_cache ---->/var/vcap/data/warden/depot/17vedc80gj3/mnt/var/vcap/packages/buildpack_cache (readonly)

/var/vcap/data/dea_next/droplets/850c48ada15591d8c6a56a0e1c64cb2846ea00dd---->
/var/vcap/data/warden/depot/17vedc80gj3/mnt/var/vcap/data/dea_next/droplets/850c48ada15591d8c6a56a0e1c64cb2846ea00dd (readonly)