LXC1.0.7-- lxc-start 源码分析 01

最近较关心LinuxContainer 的启动流程，所以就从lxc_start.c这个文件看起。

首先进入源文件，直接到main程序来，本人喜欢按照程序执行的顺序来看代码，所以看个人喜好了。

int main(int argc, char *argv[])
{
int err = 1;
struct lxc_conf *conf;    //初始化config结构
char *const *args;		 //传递的参数
char *rcfile = NULL;		 //指定配置文件
char *const default_args[] = {    //默认的args参数
"/sbin/init",
NULL,
};
struct lxc_container *c;    //lxc-container 的结构体
….
}

看下lxc_conf这个数据结构

struct lxc_conf {
int is_execute;		//容器是否在执行
char *fstab;			//fstab？
int tty;				//tty的个数
int pts;				//pts的个数？
int reboot;			//重启？
int need_utmp_watch;		//字面翻译 需要utmp 查看
signed long personality;		//字面翻译 特点
struct utsname *utsname;	//ustname
struct lxc_list cgroup;		//cgroup list lxc_list只是简单的链表结构
struct lxc_list id_map;		//id_map list
struct lxc_list network;		//network list
struct saved_nic *saved_nics;//saved_nics 结构
int num_savednics;			//savednics数量？
int auto_mounts;			//auto_mounts?
struct lxc_list mount_list;		//mount_list list?
struct lxc_list caps;			//caps list？
struct lxc_list keepcaps;		//keepcaps list?
struct lxc_tty_info tty_info;	//tty的相关信息
struct lxc_console console;	//console的结构体
struct lxc_rootfs rootfs;		//rootfs的结构体
char *ttydir;					//tty目录
int close_all_fds;			//关闭所有fd
struct lxc_list hooks[NUM_LXC_HOOKS];	//hooks 函数

char *lsm_aa_profile;   //?
char *lsm_se_context;	//?
int tmp_umount_proc;	//?
char *seccomp;  // filename with the seccomp rules
#if HAVE_SCMP_FILTER_CTX
scmp_filter_ctx seccomp_ctx;
#endif
int maincmd_fd;		//?
int autodev;  // if 1, mount and fill a /dev at start
int haltsignal; // signal used to halt container
int stopsignal; // signal used to hard stop container
int kmsg;  // if 1, create /dev/kmsg symlink
char *rcfile;   // Copy of the top level rcfile we read

// Logfile and loglevel can be set in a container config file.
// Those function as defaults.  The defaults can be overriden
// by command line.  However we don't want the command line
// specified values to be saved on c->save_config().  So we
// store the config file specified values here.
char *logfile;  // the logfile as specifed in config
int loglevel;   // loglevel as specifed in config (if any)

int inherit_ns_fd[LXC_NS_MAX];

int start_auto;
int start_delay;
int start_order;
struct lxc_list groups;
int nbd_idx;

/* set to true when rootfs has been setup */
bool rootfs_setup;
};

lxc_info 的结构体看完，下面要看下lxc-container 这个重头戏。

下面来看下lxc_container的结构体

/*!
* An LXC container.
*/
struct lxc_container {
// private fields
char *name; 		 //container 的名字
char *configfile; 		 // configuration file 的路径
char *pidfile;   		 // 存储pid 的文件名
struct lxc_lock *slock;  //Container semaphore lock. 容器的信号锁
struct lxc_lock *privlock;//容器的私有信号锁
int numthreads;			//容器的引用数量，由privlock保护
struct lxc_conf *lxc_conf;

// public fields
char *error_string;		//全局变量 可读的最后显示的error
int error_num;			//最后error的数字
bool daemonize;		//容器是否希望开启守护进程
char *config_path;		// configuration file 的路径 和上面的区别？ 全局？
…….					//一堆成员函数   暂不看
}

好了， lxc_container暂时看到这个位置，后面需要的话再一个一个的详解

 

lxc_list_init(&defines);             //初始化list

defines定义在文件开始，为全局变量

static structlxc_list defines;

if(lxc_caps_init())                    //caps初始化

        return err;

到这个函数里看一下。

int lxc_caps_init(void)
{
uid_t uid = getuid();
gid_t gid = getgid();
uid_t euid = geteuid();	//有效uid

if (!uid) {				//root权限运行的话就省了后面的步骤了
INFO("command is run as 'root'");
return 0;
}

if (uid && !euid) {
INFO("command is run as setuid root (uid : %d)", uid);

if (prctl(PR_SET_KEEPCAPS, 1)) {		//prctl 设置进程的选项，为下面set？
ERROR("failed to 'PR_SET_KEEPCAPS': %m");
return -1;
}

if (setresgid(gid, gid, gid)) {
ERROR("failed to change gid to '%d': %m", gid);
return -1;
}

if (setresuid(uid, uid, uid)) {
ERROR("failed to change uid to '%d': %m", uid);
return -1;
}

if (lxc_caps_up()) {
ERROR("failed to restore capabilities: %m");
return -1;
}
}

if (uid == euid)
INFO("command is run as user '%d'", uid);

return 0;
}

接着就是读传过来的参数

if(lxc_arguments_parse(&my_args, argc, argv))

        return err;

这个函数就没细看，只需知道将参数传给my_args

判断有没有指定 初始执行的参数，没有的话指定默认参数

if (!my_args.argc)

        args = default_args;

    else       

        args = my_args.argv;

 

初始化一堆log的，暂时也没细看

if (lxc_log_init(my_args.name, my_args.log_file, my_args.log_priority,
my_args.progname, my_args.quiet, my_args.lxcpath[0]))
return err;
lxc_log_options_no_override();

const char *lxcpath = my_args.lxcpath[0];		//lxcpath 很有意思
// lxc_global_config_value("lxc.lxcpath")这个写的还是比较复杂的，总之lxcpath会是默认的路径
//指定config的位置，如果没指定，则使用默认的路径的config，通过配置创建新的
/*
* rcfile possibilities:
* 1. rcfile from random path specified in cli option
* 2. rcfile not specified, use $lxcpath/$lxcname/config
* 3. rcfile not specified and does not exist.
*/
/* rcfile is specified in the cli option */
if (my_args.rcfile) {
rcfile = (char *)my_args.rcfile;
c = lxc_container_new(my_args.name, lxcpath);
if (!c) {
ERROR("Failed to create lxc_container");
return err;
}
c->clear_config(c);
if (!c->load_config(c, rcfile)) {
ERROR("Failed to load rcfile");
lxc_container_put(c);
return err;
}
}
} else {
int rc;

rc = asprintf(&rcfile, "%s/%s/config", lxcpath, my_args.name);
if (rc == -1) {
SYSERROR("failed to allocate memory");
return err;
}
INFO("using rcfile %s", rcfile);

/* container configuration does not exist */
if (access(rcfile, F_OK)) {
free(rcfile);
rcfile = NULL;
}
c = lxc_container_new(my_args.name, lxcpath);
if (!c) {
ERROR("Failed to create lxc_container");
return err;
}
}

里面最主要的函数c = lxc_container_new(my_args.name, lxcpath);

struct lxc_container *lxc_container_new(const char *name, const char *configpath)
{
struct lxc_container *c;			//结构体lxc_container 前面分析过了

c = malloc(sizeof(*c));			//创建
if (!c) {
fprintf(stderr, "failed to malloc lxc_container\n");
return NULL;
}
memset(c, 0, sizeof(*c));		   //初始0

if (configpath)
c->config_path = strdup(configpath);			//config_path
else
c->config_path = strdup(lxc_global_config_value("lxc.lxcpath"));

if (!c->config_path) {
fprintf(stderr, "Out of memory\n");
goto err;
}

remove_trailing_slashes(c->config_path);
c->name = malloc(strlen(name)+1);
if (!c->name) {
fprintf(stderr, "Error allocating lxc_container name\n");
goto err;
}
strcpy(c->name, name);

c->numthreads = 1;
// lock这部分没细看
if (!(c->slock = lxc_newlock(c->config_path, name))) {
fprintf(stderr, "failed to create lock\n");
goto err;
}
if (!(c->privlock = lxc_newlock(NULL, NULL))) {
fprintf(stderr, "failed to alloc privlock\n");
goto err;
}
// set config path
if (!set_config_filename(c)) {
fprintf(stderr, "Error allocating config file pathname\n");
goto err;
}
//load config path
if (file_exists(c->configfile) && !lxcapi_load_config(c, NULL))
goto err;
//判断容器是否创建失败
if (ongoing_create(c) == 2) {
ERROR("Error: %s creation was not completed", c->name);
lxcapi_destroy(c);
lxcapi_clear_config(c);
}
c->daemonize = true;
c->pidfile = NULL;
…… 				//后面都是成员函数赋值
}
现在回到lxc_start 的main函数中
//判断容器是否在运行
if (c->is_running(c)) {
ERROR("Container is already running.");
err = 0;
goto out;
}
/*
* We should use set_config_item() over &defines, which would handle
* unset c->lxc_conf for us and let us not use lxc_config_define_load()
*/
//加载config文件
if (!c->lxc_conf)
c->lxc_conf = lxc_conf_init();
co
bbfe
nf = c->lxc_conf;
if (lxc_config_define_load(&defines, conf))
goto out;
//提示信息
if (!rcfile && !strcmp("/sbin/init", args[0])) {
ERROR("Executing '/sbin/init' with no configuration file may crash the host");
goto out;
}

if (ensure_path(&conf->console.path, my_args.console) < 0) {
ERROR("failed to ensure console path '%s'", my_args.console);
goto out;
}

if (ensure_path(&conf->console.log_path, my_args.console_log) < 0) {
ERROR("failed to ensure console log '%s'", my_args.console_log);
goto out;
}
// pid 文件
if (my_args.pidfile != NULL) {
if (ensure_path(&c->pidfile, my_args.pidfile) < 0) {
ERROR("failed to ensure pidfile '%s'", my_args.pidfile);
goto out;
}
}
//一些share_ns 的配置，未细看
int i;
for (i = 0; i < LXC_NS_MAX; i++) {
if (my_args.share_ns[i] == NULL)
continue;

int pid = pid_from_lxcname(my_args.share_ns[i], lxcpath);
if (pid < 1)
goto out;

int fd = open_ns(pid, ns_info[i].proc_name);
if (fd < 0)
goto out;
conf->inherit_ns_fd[i] = fd;
}
//初始化为1
if (!my_args.daemonize) {
c->want_daemonize(c, false);
}

if (my_args.close_all_fds)
c->want_close_all_fds(c, true);

err = c->start(c, 0, args) ? 0 : 1;

if (err) {
ERROR("The container failed to start.");
if (my_args.daemonize)
ERROR("To get more details, run the container in foreground mode.");
ERROR("Additional information can be obtained by setting the "
"--logfile and --logpriority options.");
err = c->error_num;
lxc_container_put(c);
return err;
}

out:
lxc_container_put(c);
return err;
}

直接到c->start 过程start是调用 lxcapi_start 这个函数指针，现在去看下这个函数到底是怎么讲lxc container 启动起来的。

    传过来的参数是container c，useinit 0，argv=args 即指定的初始化程序

static bool lxcapi_start(struct lxc_container *c, int useinit, char * const argv[])
{
int ret;
struct lxc_conf *conf;
bool daemonize = false;				//守护进程为false
FILE *pid_fp = NULL;				//pid_file文件的指针
char *default_args[] = {				//又是default_args
"/sbin/init",
NULL,
};

/* container exists */
if (!c)								//判断容器是否存在
return false;
/* container has been setup */
if (!c->lxc_conf)						//config加载完美
return false;

if ((ret = ongoing_create(c)) < 0) {		//容器是否创建完整
ERROR("Error checking for incomplete creation");
return false;
}
if (ret == 2) {
ERROR("Error: %s creation was not completed", c->name);
c->destroy(c);
return false;
} else if (ret == 1) {
ERROR("Error: creation of %s is ongoing", c->name);
return false;
}

/* is this app meant to be run through lxcinit, as in lxc-execute? */
if (useinit && !argv)					//还是判断
return false;

if (container_mem_lock(c))			//lock
return false;
conf = c->lxc_conf;					//conf赋值
daemonize = c->daemonize;			//true
container_mem_unlock(c);		//unlock

if (useinit) {						//0
ret = lxc_execute(c->name, argv, 1, conf, c->config_path);
return ret == 0 ? true : false;
}

if (!argv)
argv = default_args;			//又重新判断 args 是否为空，空即赋值
/*
* say, I'm not sure - what locks do we want here?  Any?
* Is liblxc's locking enough here to protect the on disk
* container?  We don't want to exclude things like lxc_info
* while container is running...
* 这段注释给跪了，还是老老实实看他想干嘛吧
*/
if (daemonize) {					//true
lxc_monitord_spawn(c->config_path);	//start好像跟前面的版本差别

pid_t pid = fork();
if (pid < 0)
return false;

if (pid != 0) {
/* Set to NULL because we don't want father unlink
* the PID file, child will do the free and unlink.
*/
c->pidfile = NULL;
return wait_on_daemonized_start(c, pid);		//等下进去，里面有waitpid，所以先看后面
}

/* second fork to be reparented by init */
pid = fork();										//两次fork
if (pid < 0) {
SYSERROR("Error doing dual-fork");
return false;
}
if (pid != 0)
exit(0);
/* like daemon(), chdir to / and redirect 0,1,2 to /dev/null */
if (chdir("/")) {									//root目录
SYSERROR("Error chdir()ing to /.");
return false;
}
lxc_check_inherited(conf, -1);
close(0);					//pipe file?
close(1);
close(2);
open("/dev/zero", O_RDONLY);
open("/dev/null", O_RDWR);
open("/dev/null", O_RDWR);
setsid();
} else {
if (!am_single_threaded()) {
ERROR("Cannot start non-daemonized container when threaded");
return false;
}
}
/* We need to write PID file after daeminize, so we always
* write the right PID.
*/
if (c->pidfile) {								//写入pid 到pidfile
pid_fp = fopen(c->pidfile, "w");
if (pid_fp == NULL) {
SYSERROR("Failed to create pidfile '%s' for '%s'",
c->pidfile, c->name);
return false;
}

if (fprintf(pid_fp, "%d\n", getpid()) < 0) {
SYSERROR("Failed to write '%s'", c->pidfile);
fclose(pid_fp);
pid_fp = NULL;
return false;
}

fclose(pid_fp);
pid_fp = NULL;
}

reboot:
…..
}

现在到 wait_on_daemonized_start(c, pid) 里面看看函数调用的情况

这个就是主线程的pid 在等待其他子线程工作完，然后执行，只能硬着头皮继续看了。

static bool wait_on_daemonized_start(struct lxc_container *c, int pid)
{
/* we'll probably want to make this timeout configurable? */
int timeout = 5, ret, status;

/*
* our child is going to fork again, then exit.  reap the
* child
*/
ret = waitpid(pid, &status, 0);
if (ret == -1 || !WIFEXITED(status) || WEXITSTATUS(status) != 0)
DEBUG("failed waiting for first dual-fork child");
return lxcapi_wait(c, "RUNNING", timeout);
}

函数很简单 直接调用了lxcapi_wait。

static bool lxcapi_wait(struct lxc_container *c, const char *state, int timeout)
{
int ret;

if (!c)
return false;

ret = lxc_wait(c->name, state, timeout, c->config_path);
return ret == 0;
}

这个依旧很简单又跳走了。。。lxc_wait了

这个函数现在先不细说了，只是检查容器创建是否超时的问题。

 

Reboot这货还是很奇葩，我们这一代都在灌输少用或者不用goto，导致我见到这个一直很疑惑goto在哪，结果找了半天没找到，好吧 他是单独的一个程序块，瞅瞅去，重头戏都在这里。

ok 下篇再看reboot吧，重头戏都在里面