RBD中主要的数据结构

Dispatcher
/_\          |->conf (md_config_t *)
|           |-monclient (MonClient)
librados::Rados--client (librados::RadosClient *)--|->messenger (Messenger *)
|->objecter (Objecter *)
|-finisher (Finisher)
|-state (enum{DISCONNECTED,CONNECTING, CONNECTED})
|-instance_id (uint64_t)

SimplePolicyMessenger
/_\
|        |-accepter (Accepter)
SimpleMessenger--|-dispatch_queue (DispatchQueue)
|-global_seq (__u32)
|-dispatch_throttler (Throttle)
|-local_connection (ConnectionRef)
|-reaper_thread (ReaperThread)

Connection--|->msgr (Messenger*)
|->priv (RefCountedObject)
|-peer_type (int)
|-peer_addr (entity_addr_t)
|-last_keepalive_ack (utime_t)

Connection
/_\
|
PipConnection--|->pipe (Pipe *)

2 struct entity_addr_t {                                                                                                                                 +OSD(int i=NEW)
__u32 type;                                                                                                                                          +decode(bufferlist::it❯
__u32 nonce;                                                                                                                                         +encode(bufferlist& bl❯
union {                                                                                                                                              +entity_name_t()
¦ sockaddr_storage addr;                                                                                                                             +entity_name_t(int t, ❯
¦ sockaddr_in addr4;                                                                                                                                 +entity_name_t(const c❯
¦ sockaddr_in6 addr6;                                                                                                                                +is_client() const
};
...
}

Dispatcher
/_\
|       |->messenger (Messenger *)
MonClient--|-state (MonClientState)
|-cur_mon (string)
|-finisher (Finisher)
|->session_established_context (Context *)
|->auth (AuthClientHandler)
|->keyring (KeyRing *)
|->authorize_handler_registry (AuthAuthorizeHandlerRegistry)

Dispatcher
/_\
|     |->messenger (Messenger *)
Objecter--|->mc (MonClient *)
|->finisher (Finisher *)
|->osdmap (OSDMap *)
|->homeless_session (OSDSession *)
|-op_throttle_bytes (Throttle) //可根据底层硬件能力，调整以提高性能
|-op_throttle_ops (Throttle)

librados::IoCtx--|->io_ctx_imp (librados::IoCtxImpl *)

|-poolid (int64_t)
librados::IoCtxImpl::IoCtxImpl--|->client (RadosClient *)
|-oloc (object_locator_t)
|-aio_write_seq (ceph_tid_t == int64_t)
|->objecter (Objecter *)
|-snap_seq (snapid_t)
|-snapc (SnapContext)

|-pool (int64_t) ///< pool id
object_locator_t--|-key (string) ///< key string (if non-empty)
|-nspace (string) ///< namespace
|-hash (int64_t) ///< hash position (if >= 0)

librbd::Image::--|->ctx (image_ctx_t == void *)

|-name (string)
librbd::ImageCtx--|-snap_id (uint64_t)
|-id (string)
|->object_cacher (ObjectCacher)
|->writeback_handler (LibrbdWriteback *)
|->object_set (ObjectCacher::ObjectSet *)
|-object_map (ObjectMap)
|->aio_work_queue (ContextWQ*)
|->op_work_queue (ContextWQ*)
|-md_ctx (IoCtx)
|-data_ctx (IoCtx)
|-header (struct rbd_obj_header_ondisk)
|-layout (ceph_file_layout)
|->copyup_finisher (Finisher)
|-readahead (Readahead)
|-snapc (SnapContext)
|-snaps (vector<librados::snap_t>)
|-snap_info (map<librados::snap_t, SnapInfo>)
|-snap_ids (map<std::string, librados::snap_t>)

librbd::Rbd

struct SnapContext--|-seq (snapid_t)
|-snaps (vector<snapid_t>)

op.ops.op.op
\|/  |  |  |
ObjectOperation| \|/
\|/ | ceph_osd_op
vector(OSDOp)|
\|/
OSDOp

include/rados/librados.hpp
librados::ObjectOperation--|->impl (ObjectOperationImpl *) //ObjectOperationImpl == Objecter::ObjectOperation

|-priority (int)
osdc/Objecter.h         |-flags (int)
struct ObjectOperation--|-ops (vector<OSDOp>)
|-out_bl (vector<bufferlist*>)
|-out_handler (vector<Context*>)
|-out_rval (vector<int*>)

|-op (ceph_osd_op)
struct OSDOp--|-soid (sobject_t)
|-indata (bufferlist)
|-outdata (bufferlist)
|-rval (int32_t)

struct ceph_osd_op {
__le16 op;           /* CEPH_OSD_OP_* */
__le32 flags;        /* CEPH_OSD_OP_FLAG_* */
union {
struct {
__le64 offset, length;
__le64 truncate_size;
__le32 truncate_seq;
} __attribute__ ((packed)) extent;
struct {
__le32 name_len;
__le32 value_len;
__u8 cmp_op;       /* CEPH_OSD_CMPXATTR_OP_* */
__u8 cmp_mode;     /* CEPH_OSD_CMPXATTR_MODE_* */
} __attribute__ ((packed)) xattr;
struct {
__u8 class_len;
__u8 method_len;
__u8 argc;
__le32 indata_len;
} __attribute__ ((packed)) cls;
struct {
__le64 count;
__le32 start_epoch; /* for the pgls sequence */
} __attribute__ ((packed)) pgls;
¦   ¦   struct {
¦   ¦   __le64 snapid;
¦   ¦   } __attribute__ ((packed)) snap;
struct {
__le64 cookie;
__le64 ver;     /* no longer used */
__u8 op;    /* CEPH_OSD_WATCH_OP_* */
__u32 gen;      /* registration generation */
} __attribute__ ((packed)) watch;
struct {
__le64 cookie;
} __attribute__ ((packed)) notify;
struct {
__le64 unused;
__le64 ver;
} __attribute__ ((packed)) assert_ver;
struct {
__le64 offset, length;
__le64 src_offset;
} __attribute__ ((packed)) clonerange;
struct {
__le64 max;     /* max data in reply */
} __attribute__ ((packed)) copy_get;
struct {
__le64 snapid;
__le64 src_version;
__u8 flags;
} __attribute__ ((packed)) copy_from;
struct {
struct ceph_timespec stamp;
} __attribute__ ((packed)) hit_set_get;
struct {
__u8 flags;
} __attribute__ ((packed)) tmap2omap;
struct {
__le64 expected_object_size;
__le64 expected_write_size;
} __attribute__ ((packed)) alloc_hint;
};
__le32 payload_len;
} __attribute__ ((packed));

struct sobject_t--|-oid (object_t)
|-snap (snapid_t)

struct snapid_t--|-val (uint64_t)

struct object_t--|-name (string)

Objecter::Struct Op--|->session (OSDSession*)
|-target (op_target_t)
|-con (ConnectionRef)
|-snapid (snapid_t)
|->outbl (bufferlist*)
|->onack (Context*)
|->data_offset (int*)
|-ops (vector<OSDOp>)
|-out_bl (vector<bufferlist*>)
|-out_rval (vector<int*>)
|-out_handler (vector<Context*>)
|-tid (ceph_tid_t)

struct op_target_t--|-base_oid (object_t)
|-base_oloc (object_locator_t)
|-target_oid (object_t)
|-target_oloc (object_locator_t)
|-base_pgid (pg_t)
|-pgid (pg_t)
|-pg_num (unsigned)
|-pg_num_mask (unsigned)
|-up (vector<int>)
|-acting (vector<int>)
|-up_primary (int)
|-acting_primary (int)
|-size (int)
|-min_size (int)
|-osd (int)

OSDSession--|-osd (int)
|-con (ConnectionRef)
|-ops (map<ceph_tid_t, Op*>)
|-linger_ops (map<uint64_t, LingerOp*>)
|-command_ops (map<ceph_tid_t, CommandOp*>)

librados::Rados::init_with_context()    //使用context来初始化Rados实例
>librados::Rados::rados_create_with_context()   //创建librados::RadosClient实例
>new librados::RadosClient()

librados::Rados::connect()  //与服务端建立连接
>client->connect()
>monclient.build_initial_map()  //获取monmap
>Messenger::create_client_messenger(cct, "radosclient") //创建通信层实例messenger
>Messenger::create(cct, cct->_conf->ms_type, entity_name_t::CLIENT(),lname, nonce)
>new SimpleMessenger(cct, name, lname, nonce) //根据配置中选择的网络模式来实例化对应的实例
>messenger->set_default_policy(Messenger::Policy::lossy_client(0, CEPH_FEATURE_OSDREPLYMUX)) //设置消息层默认策略
>new (std::nothrow) Objecter(cct, messenger, &monclient,&finisher,cct->_conf->rados_mon_op_timeout,cct->_conf->rados_osd_op_timeout) //创建objecter对象
>objecter->set_balanced_budget() //根据配置，设置启用throttle来控制发出的op数
>monclient.set_messenger() //为monclient实例提供消息通信层实例
>objecter->init() //初始化objecter perfcounter 并提供能够查询正在处理的op状况的钩子
>messenger->add_dispatcher_tail(objecter/this) //为消息层实例添加Dispatcher
>messenger->start() //启动messenger实例，reaper线程。
>monclient.set_want_keys(CEPH_ENTITY_TYPE_MON | CEPH_ENTITY_TYPE_OSD)
>monclient.init()
>messenger->add_dispatcher_head(this) //把monclient添加到通信层的dispatcher中。
>finisher.start()
>monclient.authenticate(conf->client_mount_timeout) //验证客户端的权限
>objecter->start()
>_maybe_request_map() //检查是否需要拉取最新的osdmap
>monc->renew_subs() //订阅最新osdmap
>monclient.renew_subs()
>finisher.start() //到此monclient 连接完成

librados::Rados::ioctx_create(const char *name, IoCtx &io)
>rados_ioctx_create2((rados_t)client, name, &p)
>client->create_ioctx(name, &ctx) //创建IoCtxImple实例，并赋值
>lookup_pool(name) //查找指定pool的ID
>new librados::IoCtxImpl(this, objecter, poolid, CEPH_NOSNAP)
>io.io_ctx_impl = (IoCtxImpl*)p //把IoCtxImpl实例地址赋予io.io_ctx_impl

librbd::Rbd::open(IoCtx& io_ctx, Image& image, const char *name)
>open(io_ctx, image, name, NULL)
>open(io_ctx, image, name, NULL)
>ImageCtx *ictx = new ImageCtx(name, "", snap_name, io_ctx, false)
>librbd::open_image(ictx)
>ictx.init()
>detect_format(md_ctx, name, &old_format, NULL) //获取image的格式，image格式有两种，image组织结构升级，并兼容老的格式
1.新格式
>cls_client::get_id(&md_ctx, id_obj_name(name), &id) //获取image的ID
>header_oid = header_name(id) //够着header name（rbd_header.{name}）
>cls_client::get_immutable_metadata(&md_ctx, header_oid,&object_prefix, &order) //获取指定header name的不变属性
>cls_client::get_stripe_unit_count(&md_ctx, header_oid,&stripe_unit, &stripe_count) //获取image的stripe信息
>init_layout() //初始化layout
2.老格式
>header_oid = old_header_name(name); //构造header name
readahead：
>readahead.set_trigger_requests(conf->rbd_readahead_trigger_requests) //设置触发readahead的条件
>readahead.set_max_readahead_size(conf->rbd_readahead_max_bytes) //设置预读字节大小
>ictx_refresh(ictx) //获取image相关的元信息
>image.ctx = (image_ctx_t) ictx;

ObjectExtent--|-oid (object_t) //object id
|-objectno (uint64_t)
|-offset (uint64_t) //in object
|-length (uint64_t) //in object
|-truncate_size (uint64_t) //in object
|-oloc (object_locator_t) //object locator (pool etc)
|-buffer_extents (vector<pair<uint64_t,uint64_t>>)