思科VPP源码分析(feature机制分析)

基本概念

早期的VPP本身的node框架比较固定，各个node之间逻辑连接已经固化。为此新版本增加了feature机制，
每个feature是一个node，用户可以启用/停止某个或某些feature。
用户也可以自己写插件，把自定义node（自己的业务逻辑）加入到指定位置。

vnet_feature_arc_registration_t

vpp将feature分成不同的组，每组feature称为一个arc。arc中的feature按照代码指定的顺序串接起来。arc结构将记录这组feature的起始node和结束node。系统初始化时完成每个feature的连接。

VNET_FEATURE_ARC_INIT宏用来注册arc。

在arc中指定的起始node中，必须调用vnet_feature_arc_start函数，才能正式进入feature机制业务流程，该函数会将下一跳强行指定为arc中的下一个feature。

vnet_feature_registration_t

一个feature等价于一个node，用户通过VNET_FEATURE_INIT宏定义自己的feature，指定需要加入哪个arc，以及在arc中的哪个相对位置。每个feature都可以通过外部命令行启用/停止。

关键函数

VNET_FEATURE_ARC_INIT和VNET_FEATURE_ARC_INIT宏将arc和feature保存到全局变量feature_main中。此时arc和feature还未被”加工”，只是基本的原料。

clib_error_t *vnet_feature_init

将把arc和feature初始化，并组织成最终的数据结构填充到vnet_feature_main_t feature_main中，代码比较简单。这里只对vnet_feature_main_t 描述即可。

typedef struct
{
/** feature arc configuration list */
//VNET_FEATURE_ARC_INIT宏中注册的arc保存在这里，顺序不定，也没有意义。
vnet_feature_arc_registration_t *next_arc;
/*vnet_feature_init()中来初始化该成员，把arc按名字组织成hash表,为何要用双重指针？？？
博主认为是bug，后面明明是当hash表用么。又不是hash数组*/
uword **arc_index_by_name;

/** feature path configuration lists */
VNET_FEATURE_ARC_INIT宏中注册的feature保存在这里，顺序不定，也没有意义。
vnet_feature_registration_t *next_feature;
/*vnet_feature_init()中来初始化该成员，把feature分配到不同arc中，并且根据
before和after变量来决定feature顺序。这里看作是一个数组，用arc的index做索引*/
vnet_feature_registration_t **next_feature_by_arc;
/*vnet_feature_init()中来初始化该成员，把feature按名字组织成hash表数组，arc的index索引到
对应hash表，node name来做hash表的key*/
uword **next_feature_by_name;

/** feature config main objects */
vnet_feature_config_main_t *feature_config_mains;

/** Save partial order results for show command */
char ***feature_nodes;

/** bitmap of interfaces which have driver rx features configured */
/*bitmap数组，arc的index索引对应bitmap表,决定该网卡是否启用了该arc中的feature*/
uword **sw_if_index_has_features;

/** feature reference counts by interface */
i16 **feature_count_by_sw_if_index;

/** Feature arc index for device-input */
/*从收包开始就使用feature机制了，这个变量给收包驱动用。追踪了下代码，这个变量属于device_input_node这个node，但是这个node业务函数为空，而且状态是VLIB_NODE_STATE_DISABLED。那这是干什么用的呢？层层追踪，原来这个node纯粹是个dump node，但是它是一个arc起点。这个arc的index保存在了下面变量中。然后真正的网卡收包node中会借用这个index当成自己的，调用vnet_feature_start_device_input_x1这类函数，开始了feature之旅。真TMD恶心，代码还能写的更难懂点么。*/
u8 device_input_feature_arc_index;

/** convenience */
vlib_main_t *vlib_main;
vnet_main_t *vnet_main;
} vnet_feature_main_t;

vnet_feature_arc_init 把属于某arc的无序的feature排序成有序状态。feature注册时可以指定在某些node前面，和某些node后面。

clib_error_t *
vnet_feature_arc_init (vlib_main_t * vm,
vnet_config_main_t * vcm,
char **feature_start_nodes,
int num_feature_start_nodes,
vnet_feature_registration_t * first_reg,
char ***in_feature_nodes)
{
uword *index_by_name;
uword *reg_by_index;
u8 **node_names = 0;
u8 *node_name;
char **these_constraints;
char *this_constraint_c;
u8 **constraints = 0;
u8 *constraint_tuple;
u8 *this_constraint;
u8 **orig, **closure;
uword *p;
int i, j, k;
u8 *a_name, *b_name;
int a_index, b_index;
int n_features;
u32 *result = 0;
vnet_feature_registration_t *this_reg = 0;
char **feature_nodes = 0;
hash_pair_t *hp;
u8 **keys_to_delete = 0;

index_by_name = hash_create_string (0, sizeof (uword));
reg_by_index = hash_create (0, sizeof (uword));

this_reg = first_reg;

/* pass 1, collect feature node names, construct a before b pairs */
while (this_reg)
{
node_name = format (0, "%s%c", this_reg->node_name, 0);
//vec_len (node_names) 指的是当前node_names数组中元素个数
hash_set (reg_by_index, vec_len (node_names), (uword) this_reg);

//现在feature用index来代表了
hash_set_mem (index_by_name, node_name, vec_len (node_names));

vec_add1 (node_names, node_name);

//runs_before是个数组，可以保存有多个node。
these_constraints = this_reg->runs_before;
while (these_constraints && these_constraints[0])
{
this_constraint_c = these_constraints[0];

constraint_tuple = format (0, "%s,%s%c", node_name,
this_constraint_c, 0);
vec_add1 (constraints, constraint_tuple);
these_constraints++;
}

//runs_after是个数组，可以保存有多个node。
these_constraints = this_reg->runs_after;
while (these_constraints && these_constraints[0])
{
this_constraint_c = these_constraints[0];

constraint_tuple = format (0, "%s,%s%c",
this_constraint_c, node_name, 0);
vec_add1 (constraints, constraint_tuple);
these_constraints++;
}

this_reg = this_reg->next;
}
//至此，把feature之间顺序关系表达为"A,B"字符串，代表A在B前面。

//其实就是该arc中所有feature个数
n_features = vec_len (node_names);
//可以看作生成一个(n_features X n_features)的二维数组
orig = clib_ptclosure_alloc (n_features);

for (i = 0; i < vec_len (constraints); i++)
{
this_constraint = constraints[i];

if (comma_split (this_constraint, &a_name, &b_name))
return clib_error_return (0, "comma_split failed!");

p = hash_get_mem (index_by_name, a_name);
/*
* Note: the next two errors mean that something is
* b0rked. As in: if you code "A depends on B," and you forget
* to define a FEATURE_INIT macro for B, you lose.
* Nonexistent graph nodes are tolerated.
*/
if (p == 0)
return clib_error_return (0, "feature node '%s' not found", a_name);
a_index = p[0];

p = hash_get_mem (index_by_name, b_name);
if (p == 0)
return clib_error_return (0, "feature node '%s' not found", b_name);
b_index = p[0];

/* add a before b to the original set of constraints */
orig[a_index][b_index] = 1;
vec_free (this_constraint);
}

/* Compute the positive transitive closure of the original constraints */
closure = clib_ptclosure (orig);

/* Compute a partial order across feature nodes, if one exists. */
again:
for (i = 0; i < n_features; i++)
{
for (j = 0; j < n_features; j++)
{
if (closure[i][j])
goto item_constrained;
}
/* Item i can be output */
vec_add1 (result, i);
{
for (k = 0; k < n_features; k++)
closure[k][i] = 0;
/*
* Add a "Magic" a before a constraint.
* This means we'll never output it again
*/
closure[i][i] = 1;
goto again;
}
item_constrained:
;
}

/* see if we got a partial order... */
if (vec_len (result) != n_features)
return clib_error_return (0, "%d feature_init_cast no partial order!");
//到这里，feature的顺序关系就计算成功了。保存在result中，每个feature用index代表。
/*
* We win.
* Bind the index variables, and output the feature node name vector
* using the partial order we just computed. Result is in stack
* order, because the entry with the fewest constraints (e.g. none)
* is output first, etc.
*/

//把feature索引按顺序转换成对应的vnet_feature_registration_t信息，保存在feature_nodes中
for (i = n_features - 1; i >= 0; i--)
{
p = hash_get (reg_by_index, result[i]);
ASSERT (p != 0);
this_reg = (vnet_feature_registration_t *) p[0];
if (this_reg->feature_index_ptr)
*this_reg->feature_index_ptr = n_features - (i + 1);
this_reg->feature_index = n_features - (i + 1);
vec_add1 (feature_nodes, this_reg->node_name);
}

/* Set up the config infrastructure */
/*feature顺序现在保存在feature_nodes中，现在要把这个关系保存到vnet_config_main_t中了。
注意每个arc都有一份自己的vnet_config_main_t*/
vnet_config_init (vm, vcm,
feature_start_nodes,
num_feature_start_nodes,
feature_nodes, vec_len (feature_nodes));

/* Save a copy for show command */
*in_feature_nodes = feature_nodes;

/* Finally, clean up all the shit we allocated */
/* *INDENT-OFF* */
hash_foreach_pair (hp, index_by_name,
({
vec_add1 (keys_to_delete, (u8 *)hp->key);
}));
/* *INDENT-ON* */
hash_free (index_by_name);
for (i = 0; i < vec_len (keys_to_delete); i++)
vec_free (keys_to_delete[i]);
vec_free (keys_to_delete);
hash_free (reg_by_index);
vec_free (result);
clib_ptclosure_free (orig);
clib_ptclosure_free (closure);
return 0;
}

至此，feature已经计算好了顺序，但是还是没有真正把各个feature连接起来。连接各个feature的真正工作，最终交由vnet_config_add_feature()和vnet_config_del_feature()完成。

u32
vnet_config_add_feature (vlib_main_t * vm,
vnet_config_main_t * cm,
u32 config_string_heap_index,
u32 feature_index,
void *feature_config, u32 n_feature_config_bytes)
{
vnet_config_t *old, *new;
vnet_config_feature_t *new_features, *f;
u32 n_feature_config_u32s;
u32 node_index = vec_elt (cm->node_index_by_feature_index, feature_index);

if (node_index == ~0)        // feature node does not exist
return config_string_heap_index;    // return original config index

if (config_string_heap_index == ~0)
{
old = 0;
new_features = 0;
}
else
{
u32 *p = vnet_get_config_heap (cm, config_string_heap_index);
old = pool_elt_at_index (cm->config_pool, p[-1]);
new_features = old->features;
if (new_features)
new_features = duplicate_feature_vector (new_features);
}

//现有的feature基础上增加参数中传来的新feature。注意每个feature都是这么一个个添加进来的。
vec_add2 (new_features, f, 1);
f->feature_index = feature_index;
f->node_index = node_index;

//不同feature可能有自己独特的config内容，因此参数中的feature_config和n_feature_config_bytes
就是用来干这事的，保存在f->feature_config中，没有私有内容，则计算出来为0。
n_feature_config_u32s =
round_pow2 (n_feature_config_bytes,
sizeof (f->feature_config[0])) /
sizeof (f->feature_config[0]);
vec_add (f->feature_config, feature_config, n_feature_config_u32s);

/* Sort (prioritize) features. */
//上文所知，feature的index也代表着在arc中的顺序，这里排下序。
if (vec_len (new_features) > 1)
vec_sort_with_function (new_features, feature_cmp);

if (old)
remove_reference (cm, old);

//开始干活了，连接每个feature。config内存也会更新
new = find_config_with_features (vm, cm, new_features);
new->reference_count += 1;

/*
* User gets pointer to config string first element
* (which defines the pool index
* this config string comes from).
*/
vec_validate (cm->config_pool_index_by_user_index,
new->config_string_heap_index + 1);
cm->config_pool_index_by_user_index[new->config_string_heap_index + 1]
= new - cm->config_pool;
return new->config_string_heap_index + 1;
}

u32
vnet_config_del_feature (vlib_main_t * vm,
vnet_config_main_t * cm,
u32 config_string_heap_index,
u32 feature_index,
void *feature_config, u32 n_feature_config_bytes)
{
vnet_config_t *old, *new;
vnet_config_feature_t *new_features, *f;
u32 n_feature_config_u32s;

{
/*每个feature组合都有一个对应的config内存。*/
u32 *p = vnet_get_config_heap (cm, config_string_heap_index);
/*config内存第一个32bit，保存了该config的索引号。这代码很恶心，作者为啥不弄个结构出来，
还好懂些。*/
old = pool_elt_at_index (cm->config_pool, p[-1]);
}

n_feature_config_u32s =
round_pow2 (n_feature_config_bytes,
sizeof (f->feature_config[0])) /
sizeof (f->feature_config[0]);

/* Find feature with same index and opaque data. */
//找到要删除的目标feature
vec_foreach (f, old->features)
{
if (f->feature_index == feature_index
&& vec_len (f->feature_config) == n_feature_config_u32s
&& (n_feature_config_u32s == 0
|| !memcmp (f->feature_config, feature_config,
n_feature_config_bytes)))
break;
}

/* Feature not found. */
if (f >= vec_end (old->features))
return config_string_heap_index;    // return original config index

//生成一组新feature，不包含目标feature。
new_features = duplicate_feature_vector (old->features);
f = new_features + (f - old->features);
vnet_config_feature_free (f);
vec_delete (new_features, 1, f - new_features);

/* must remove old from config_pool now as it may be expanded and change
memory location if the following function find_config_with_features()
adds a new config because none of existing config's has matching features
and so can be reused */
remove_reference (cm, old);
//新生成的feature重新连接下，config内存也会更新
new = find_config_with_features (vm, cm, new_features);
new->reference_count += 1;

vec_validate (cm->config_pool_index_by_user_index,
new->config_string_heap_index + 1);
cm->config_pool_index_by_user_index[new->config_string_heap_index + 1]
= new - cm->config_pool;
return new->config_string_heap_index + 1;
}

static vnet_config_t *
find_config_with_features (vlib_main_t * vm,
vnet_config_main_t * cm,
vnet_config_feature_t * feature_vector)
{
u32 last_node_index = ~0;
vnet_config_feature_t *f;
u32 *config_string;
uword *p;
vnet_config_t *c;

config_string = cm->config_string_temp;
cm->config_string_temp = 0;
if (config_string)
_vec_len (config_string) = 0;

vec_foreach (f, feature_vector)
{
/* Connect node graph. */
//按顺序连接各个node
f->next_index = add_next (vm, cm, last_node_index, f->node_index);
last_node_index = f->node_index;

/* Store next index in config string. */
//下一个node在本node中的slot号，保存进config_string
vec_add1 (config_string, f->next_index);

/* Store feature config. */
//slot号后面还可以保存本node特有的config信息，可以为0大小
vec_add (config_string, f->feature_config, vec_len (f->feature_config));
}

/* Terminate config string with next for end node. */
//确保最终连接到end node
if (last_node_index == ~0 || last_node_index != cm->end_node_index)
{
u32 next_index = add_next (vm, cm, last_node_index, cm->end_node_index);
vec_add1 (config_string, next_index);
}

/* See if config string is unique. */
//看看hash表里是不是有同样的config_string，注意这里比较的是config_string内容，不是地址
p = hash_get_mem (cm->config_string_hash, config_string);
if (p)
{
/* Not unique.  Share existing config. */
//大多数情况下config_string不会重复，除非旧的还被别的模块引用了没释放，那这里就继续引用旧的释放新的
cm->config_string_temp = config_string;   /* we'll use it again later. */
free_feature_vector (feature_vector);
c = pool_elt_at_index (cm->config_pool, p[0]);
}
else
{
u32 *d;

//分配新的config结构
pool_get (cm->config_pool, c);
c->index = c - cm->config_pool;
c->features = feature_vector;
c->config_string_vector = config_string;

/* Allocate copy of config string in heap.
VLIB buffers will maintain pointers to heap as they read out
configuration data. */
c->config_string_heap_index
= heap_alloc (cm->config_string_heap, vec_len (config_string) + 1,
c->config_string_heap_handle);

/* First element in heap points back to pool index. */
d =
vec_elt_at_index (cm->config_string_heap,
c->config_string_heap_index);
//注意这里，config第一个4字节保存的是config索引号，之后才是slot号-私有config-slot号-私有config ......
d[0] = c->index;
clib_memcpy (d + 1, config_string, vec_bytes (config_string));
hash_set_mem (cm->config_string_hash, config_string, c->index);

c->reference_count = 0;   /* will be incremented by caller. */
}

return c;
}

现在各个feature node已经连接好了，连接信息也保存到了config中，接下来就是在业务node中使用了。

在需要开始使用feature机制的业务node中调用如下函数即可，注意该业务node必须是arc中的起始node

static_always_inline void
vnet_feature_arc_start (u8 arc, u32 sw_if_index, u32 * next0,
vlib_buffer_t * b0)
{
vnet_feature_arc_start_with_data (arc, sw_if_index, next0, b0, 0);
}

此函数会将next0修改为下一跳feature。feature的顺序在上文中已经确定

获取下一跳feature的逻辑很简单，在函数：

always_inline void *
vnet_get_config_data (vnet_config_main_t * cm,
u32 * config_index, u32 * next_index, u32 n_data_bytes)
{
u32 i, n, *d;

//vlib_buffer_t->current_config_index用来记录执行到了哪个feature了，注意最初值为1。
i = *config_index;

d = heap_elt_at_index (cm->config_string_heap, i);

n = round_pow2 (n_data_bytes, sizeof (d[0])) / sizeof (d[0]);

/* Last 32 bits are next index. */
//本feature下一跳的slot号
*next_index = d
;

/* Advance config index to next config. */
//记录下一跳feature的config索引
*config_index = (i + n + 1);

/* Return config data to user for this feature. */
return (void *) d;
}