存储之磁盘读写-----基于scsi cmd和bio

            这几天做了一个抽象磁盘驱动的例子，直接提供用户ioctl接口，把用户态buf（页对齐）写入（读取）到磁盘，并使用了两个方案做性能测试对比：一种scsi层，一种块层；

基于scsi   cdb  读写硬盘-----------裸盘读写

#include <linux/fs.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/cdev.h>
#include <linux/device.h>
#include <asm/uaccess.h>
#include <linux/semaphore.h>
#include <linux/spinlock.h>
#include <linux/delay.h>
#include <linux/scatterlist.h>
#include <scsi/scsi.h>
#include <scsi/scsi_host.h>
#include <scsi/scsi_device.h>
#include <scsi/scsi_transport.h>
#include <scsi/scsi_cmnd.h>
#include <scsi/scsi_dbg.h>
#include <scsi/scsi_eh.h>
#include <linux/list.h>
#define NAME "disk_test_io"
//ioctrl cmd
#define DISK_OPEN 1000
#define DISK_READ 1001
#define DISK_WRITE 1002
#define DISK_DONE 1003
#define DISK_SYNC 1004
#define MAX_FREQUEUE_COUNT 1024
#define MAX_DONEQUEUE_COUNT 16
//erro no
#define ERR_NO_DEVICE -1
#define ERR_BUSY -2
#define ERR_PARAM -3
typedef unsigned int uint32;
typedef unsigned long long uint64;
typedef unsigned char uint8;
static int chardev_var = 0;
//static int chardev_count = 0;
static struct semaphore sem;
static spinlock_t spin;
static struct cdev diskio_cdev;
static struct class *diskio_class;
static struct device *diskio_device;
static int device_major = 0;
struct scsi_device *g_sdev[1024]={NULL};
struct deviceid {
uint32 hostnum;
uint32 channel;
uint32 id;
uint32 lun;
int fd;
};
struct iocmd_head{
uint32 fd; //sdev[index] --> index = fd
char *buf;
uint32 len; // x*4k
uint64 offset; // x*512 sector
uint64 seq_num;
};
struct cmd_result{

int result;
uint64 seq_num;
struct scsi_sense_hdr sshr;
char cdb[10];
int nr_page;
struct scatterlist sgl[128];
struct page* pages[128];
struct list_head free_list;
struct list_head done_list;
};
struct queue_head{
spinlock_t lock;
struct list_head list;
int count;
};
struct queue_head free_queue_head;
struct queue_head done_queue_head;
DECLARE_COMPLETION_ONSTACK(com);
#define del_queue(head, member) ({ \
struct cmd_result *pos,*n; \
if(!list_empty(&head.list)){ \
spin_lock(&head.lock); \
list_for_each_entry_safe(pos, n, &head.list, member) \
kfree(pos); \
head.count=0;\
spin_unlock(&head.lock); \
} \
})
#define get_one_node(head, member) ({ \
struct cmd_result *p = NULL;\
spin_lock(&(head).lock);\
if(list_empty(&head.list)) {\
spin_unlock(&head.lock);\
p = NULL;\
}\
else {\
p = container_of(head.list.next, struct cmd_result, member);\
list_del(head.list.next);\
(head).count--;\
spin_unlock(&(head).lock);}\
p;\
})

#define add_one_node(new, head, member) ({ \
spin_lock(&head.lock);\
head.count++;\
list_add(&(new)->member,&head.list);\
spin_unlock(&head.lock);\
})
void init_queue_head(struct queue_head * q)
{
INIT_LIST_HEAD(&q->list);
q->count = 0;
spin_lock_init(&q->lock);
}
void init_free_queue(void)
{
int i;
struct cmd_result *res = NULL;
for(i = 0; i < MAX_FREQUEUE_COUNT; i++) {
res = kzalloc(sizeof(struct cmd_result), GFP_KERNEL);
if(!res){printk("kzalloc error ! \n");}
add_one_node(res, free_queue_head, free_list);
}
}
int io_open(struct deviceid *dev_id){

struct Scsi_Host *shost=NULL;
struct scsi_device *sdev=NULL;
int index=0;
printk("kernel dev_id struct { %d, %d,%d,%d }\n",dev_id->hostnum,dev_id->channel,dev_id->id,dev_id->lun);
shost=scsi_host_lookup(dev_id->hostnum);
if(shost){
sdev=scsi_device_lookup(shost,dev_id->channel,dev_id->id,dev_id->lun);
if(sdev){
for(index=0;index<1024;index++){ //this sdev already exists
if(g_sdev[index]&&(g_sdev[index]==sdev))
return index;
}
for(index=0;index<1024;index++){ //this sdev is new ,so add to g_sdev
if(g_sdev[index]==NULL){
g_sdev[index]=sdev;
printk("the new sdev=0x%p,\n",sdev);
return index;
}
}
}else{
printk("scsi device %d : %d: %d :%d not find ! --------%s--%d ! ",dev_id->hostnum,dev_id->channel,dev_id->id,dev_id->lun, __FUNCTION__,__LINE__);
}
}
else{
printk("scsi host %d not find !---%s--%d --!\n", dev_id->hostnum,__FUNCTION__,__LINE__);
}
return ERR_NO_DEVICE; //ERR_NO_DEVICE = -1 will be change
}

void diskio_done(struct scsi_cmnd *scmd ){
struct cmd_result *res = (struct cmd_result*)scmd->request;
printk("diskio_done !\n");
if (res == NULL)
goto out;
res->result = scmd->result;
for(i=0;i<res->nr_page;i++)
page_cache_release(res->pages[i]);
scsi_normalize_sense(scmd->sense_buffer, sizeof(struct scsi_sense_hdr), &res->sshr);
printk("scsi done seq num %lld\n", res->seq_num);
printk("response_code %d, sense_key %d, asc %d, ascq %d\n", res->sshr.response_code, res->sshr.sense_key, res->sshr.asc, res->sshr.ascq);
add_one_node(res, done_queue_head, done_list);
printk("execute scsi cmd return result=%x \n",res->result);
printk("frequeue count = %d, done queue count == %d\n", free_queue_head.count, done_queue_head.count);
out:
complete(&com);
scsi_put_command(scmd);
}
static int do_io(int cmd , int fd, char * buf, uint32 len, uint64 offset, uint64 seq_num)
{
int ret=0;
int nr=0;
int i=0;
unsigned long flags;
int cdb_len=0;
struct scsi_cmnd *scmd;
unsigned char *cdb;
struct page ** pages;
struct scatterlist *sg;
uint32 offset_h=(uint32)((offset&0xffffffff00000000)>>32);
uint32 offset_l=(uint32)((offset&0x00000000ffffffff));
int nr_pages=((uint64)buf+len+PAGE_SIZE-1)/PAGE_SIZE-(uint64)buf/PAGE_SIZE;

struct cmd_result *res = get_one_node(free_queue_head, free_list);
if (!res) return ERR_BUSY; //ERR_BUSY = -2
cdb= res->cdb;
pages= res->pages;
sg= res->sgl;;
res->seq_num = seq_num;
res->nr_page=nr_pages;
if(!g_sdev[fd]){
printk("g_sdev \n");
return ERR_PARAM; ////ERR_PARAM= -3
}
if((len%512) || ( len > 128*512)) {
printk("buf length should be x*512 !");
return ERR_PARAM; //ERR_PARAM= -3
};
if(offset < 0xffffffff){
if(cmd==DISK_READ) {
cdb[0]=0x28; //operation code 28 read(10)
}else if(cmd==DISK_WRITE){
cdb[0]=0x2A; //operation code 2A write(10)
}
cdb[1]=0x0; //FUA
cdb[2]=(uint8)((offset&0xff000000)>>24); //LBA logical block bufess cdb[2-5] offset
cdb[3]=(uint8)((offset&0x00ff0000)>>16);
cdb[4]=(uint8)((offset&0x0000ff00)>>8);
cdb[5]=(uint8)( offset&0x000000ff);
cdb[6]=0; //group number
cdb[7]=(uint8)((((len+512-1)/512)&0x0000ff00) >>8); //transfer number of sector cdb[7-8]
cdb[8]=(uint8)(((len+512-1)/512)&0x000000ff);
cdb[9]=0; //control
cdb_len=10;
for(i=0;i<10;i++)
printk("cdb[%d]= 0x%x ==%d !\n",i,cdb[i],cdb[i]);
}else{ //offset >0xffffffff
if(cmd==DISK_READ) {
cdb[0]=0x88; //operation code 28 read(16)
}else if(cmd==DISK_WRITE){
cdb[0]=0x8A; //operation code 8A write(16)
}
cdb[1]=0x0; //FUA
cdb[2]=(uint8)((offset_h&0xff000000)>>24); //LBA logical block bufess cdb[2-9] offset
cdb[3]=(uint8)((offset_h&0x00ff0000)>>16);
cdb[4]=(uint8)((offset_h&0x0000ff00)>>8);
cdb[5]=(uint8)( offset_h&0x000000ff);
cdb[6]=(uint8)((offset_l&0xff000000)>>24);
cdb[7]=(uint8)((offset_l&0x00ff0000)>>16);
cdb[8]=(uint8)((offset_l&0x0000ff00)>>8);
cdb[9]=(uint8)( offset_l&0x000000ff);
cdb[10]=(uint8)((((len+512-1)/512)&0xff000000) >>24); //transfer number of sector cdb[10-13]
cdb[11]=(uint8)((((len+512-1)/512)&0x00ff0000)>>16);
cdb[12]=(uint8)((((len+512-1)/512)&0x0000ff00)>>8);
cdb[13]=(uint8)(((len+512-1)/512)&0x000000ff);
cdb[14]=0; //group number
cdb[15]=0; //control
cdb_len=16;
for(i=0;i<16;i++)
printk("cdb[%d]= 0x%x ==%d !\n",i,cdb[i],cdb[i]);
}
printk("enter the do_io :cmd=%d , fd= %d , buf= 0X%p ,len =%d, offset= %lld \n",cmd,fd, buf,len,offset);
down_read(¤t->mm->mmap_sem);
ret=get_user_pages(current, current->mm, (unsigned long)buf, nr_pages, 1, 0, pages, NULL);
up_read(¤t->mm->mmap_sem);
if(ret<0) {
printk("get_user_pages failed ! %d ",nr_pages);
return ERR_PARAM; //ERR_PARAM= -3
}
printk("g_sdev[fd] = 0X%p \n",g_sdev[fd]);
scmd=scsi_get_command(g_sdev[fd],GFP_ATOMIC);
if(unlikely(!scmd)){
printk("scmd is NULL ! \n");
return ERR_BUSY; // ERR_BUSY=-2
}
printk("g_sdev[fd]= 0x%p , scmd is 0X%p ,scmd->device=0x%p ! \n",g_sdev[fd],scmd,scmd->device);
scmd->cmnd=cdb;
scmd->cmd_len=cdb_len;
scmd->sdb.length=len; //
scmd->sdb.resid=0; //do not know how to use !
scmd->sdb.table.nents=nr_pages;
scmd->sdb.table.orig_nents=nr_pages;

scmd->sdb.table.sgl=sg;
for(nr=0;nr<nr_pages-1;nr++)
{
scmd->sdb.table.sgl[nr].length=PAGE_SIZE;
scmd->sdb.table.sgl[nr].page_link = (unsigned long )pages[nr]; //page buf
scmd->sdb.table.sgl[nr].offset=0; //buf offset
scmd->sdb.table.sgl[nr].dma_address = page_to_phys(pages[nr]);
scmd->sdb.table.sgl[nr].dma_length = PAGE_SIZE;
}
scmd->sdb.table.sgl[nr].length=((len%PAGE_SIZE)?(len%PAGE_SIZE):PAGE_SIZE);
scmd->sdb.table.sgl[nr].page_link = (unsigned long )pages[nr];
scmd->sdb.table.sgl[nr].offset=0;
scmd->sdb.table.sgl[nr].dma_address = page_to_phys(pages[nr]);
scmd->sdb.table.sgl[nr].dma_length = ((len%PAGE_SIZE)?(len%PAGE_SIZE):PAGE_SIZE);;
sg_mark_end(&(scmd->sdb.table.sgl[nr]));
if(cmd==DISK_READ) {
scmd->sc_data_direction= DMA_FROM_DEVICE ; //DMA_TO_DEVICE
} else if(cmd==DISK_WRITE){
scmd->sc_data_direction= DMA_TO_DEVICE ;
}
scmd->transfersize= 512; //sector
scmd->request = (struct request*)res;
printk("start to queuecommand ! \n");

/*char *buffer=(char *)kmalloc(1024,GFP_KERNEL);
struct scsi_sense_hdr sshdr;
ret = scsi_execute_req(g_sdev[fd],cdb,scmd->sc_data_direction,buffer,512,&sshdr,6*HZ,3,NULL);
printk("scsi_execute ret= %d read buffer= %s\n",ret,buffer);
printk("sshdr {0x%x, 0x%x, %x, %x, %x, %x, %x, %x} !\n",sshdr.response_code,sshdr.sense_key,sshdr.asc,sshdr.ascq,sshdr.byte4,sshdr.byte4,sshdr.byte5,sshdr.byte6,sshdr.additional_length);
*/
//if(!(g_sdev[fd]->host)){printk("g_sdev[fd]->host \n"); return -1;}
// if(!(g_sdev[fd]->host->hostt)){printk("g_sdev[fd]->host->hostt \n"); return -1;}
// if(!(g_sdev[fd]->host->hostt->queuecommand)){printk("g_sdev[fd]->host->queuecommand \n"); return -1;}
if(g_sdev[fd]->sdev_state !=SDEV_RUNNING){
printk("this scsi device 0x%p not running !",g_sdev[fd]);
return ERR_BUSY; //ERR_BUSY=-2
}
spin_lock_irqsave(scmd->device->host->host_lock,flags);
printk("spink lock !\n");
ret=scmd->device->host->hostt->queuecommand(scmd,diskio_done);
//ret=g_sdev[fd]->host->hostt->queuecommand(scmd, diskio_done);
spin_unlock_irqrestore(scmd->device->host->host_lock,flags);
printk("queuecommand return %x --%d \n",ret,ret);

return ret; //will be change
}

static int do_sync(int cmd,int fd,uint64 seq_num){
unsigned char *cdb = NULL;
int ret=0;
int i=0;
unsigned long flags;
struct scsi_cmnd *scmd;
struct page ** pages;
struct scatterlist *sg;
struct cmd_result *res= get_one_node(free_queue_head, free_list);
if (!res) return ERR_BUSY; //ERR_BUSY=-2
pages = res->pages;
sg=res->sgl;
cdb = res->cdb;
res->seq_num = seq_num;
if(cmd!=DISK_SYNC){
printk("do_sync error !\n");
return ERR_PARAM; //ERR_PARAM=-3
}
cdb[0]=0x35;
for(i=1;i<10;i++){
cdb[i]=0;
}
printk("cmd= %d \n",cmd);
for(i=0;i<10;i++)
printk("cdb[%d]= 0x%x ==%d !\n",i,cdb[i],cdb[i]);
scmd=scsi_get_command(g_sdev[fd],GFP_ATOMIC);
if(unlikely(!scmd)){
printk("scmd is NULL ! \n");
return ERR_NO_DEVICE; //ERR_NO_DEVICE = -1
}
printk("g_sdev[fd]= 0x%p , scmd is 0X%p ,scmd->device=0x%p ! \n",g_sdev[fd],scmd,scmd->device);
scmd->cmnd=cdb;
scmd->cmd_len=10;
//scmd->sdb=NULL;
/*scmd->sdb.length=0; //
scmd->sdb.resid=0; //do not know how to use !
scmd->sdb.table.nents=0;
scmd->sdb.table.orig_nents=0;
scmd->sdb.table.sgl=NULL;
scmd->sc_data_direction= DMA_TO_DEVICE ;
*/
scmd->request = (struct request*)res;
// if(!(g_sdev[fd]->host)){printk("g_sdev[fd]->host \n"); return -1;}
// if(!(g_sdev[fd]->host->hostt)){printk("g_sdev[fd]->host->hostt \n"); return -1;}
// if(!(g_sdev[fd]->host->hostt->queuecommand)){printk("g_sdev[fd]->host->queuecommand \n"); return -1;}
if(g_sdev[fd]->sdev_state !=SDEV_RUNNING){
printk("this scsi device 0x%p not running !",g_sdev[fd]);
return ERR_BUSY; //ERR_BUSY=-2
}
spin_lock_irqsave(scmd->device->host->host_lock,flags);
printk("spink lock !\n");
ret=scmd->device->host->hostt->queuecommand(scmd,diskio_done);
//ret=g_sdev[fd]->host->hostt->queuecommand(scmd, diskio_done);
spin_unlock_irqrestore(scmd->device->host->host_lock,flags);
printk("queuecommand return %x --%d \n",ret,ret);
return ret; //will be change
}

static int userio_done(unsigned char* buf, int cnt)
{
struct user_result {
int result;
uint64 seq_num;
struct scsi_sense_hdr sshr;
};
struct cmd_result *result;
int i,ret, count = 0;
int MAX_NUM = cnt < MAX_DONEQUEUE_COUNT? cnt: MAX_DONEQUEUE_COUNT;
MAX_NUM= MAX_NUM <done_queue_head.count? MAX_NUM:done_queue_head.count;
wait_for_completion(&com);
for(i = 0; i < MAX_NUM; i++) {
result = get_one_node(done_queue_head, done_list);
if(result<=0) {
printk("userio done get node error ! \n");
continue;
}
count++;
ret = copy_to_user((unsigned char*)(buf + i * sizeof(struct user_result)), result, sizeof(struct user_result));
printk("write to user space result result %d, seq_num %lld\n", result->result, result->seq_num);
memset(result, 0, sizeof(struct cmd_result));
add_one_node(result, free_queue_head, free_list);
if (ret < 0)
return ERR_PARAM; //ERR_PARAM=-3
}
return count;
}
long diskio_ioctl (struct file *filp,unsigned int cmd, unsigned long arg)
{
int fd;
int res=0;
int ret_command=-1;
struct deviceid devid;
struct iocmd_head cmd_head;
struct {
unsigned char* buf;
int count;
} user_buf;
printk(" diskio_ioctl cmd= %d \n",cmd);
switch(cmd) {
case DISK_OPEN: //cmd=0 open to scsi device
res = copy_from_user((void *)&devid, (void __user*)arg, sizeof(devid));
if(!res){ //res =0 copy_from_user success
printk("kernel devid struct { %d,%d,%d ,%d} \n",devid.hostnum,devid.channel,devid.id,devid.lun);
fd = io_open(&devid);
devid.fd=fd;
res = copy_to_user((void __user*)arg, (void *)&devid, sizeof(devid));
return fd; // return index >=0
}
break;
case DISK_READ: //cmd=1001 read scsi device that in g_sdev[fd]
res=copy_from_user((void *)&cmd_head, (void __user*)arg, sizeof(cmd_head));
if(!res){
printk("kernel read cmd_head struct { %d, 0x%p,%d, %lld } \n",cmd_head.fd,cmd_head.buf,cmd_head.len,cmd_head.offset);
ret_command=do_io( DISK_READ, cmd_head.fd, cmd_head.buf, cmd_head.len, cmd_head.offset, cmd_head.seq_num);
}
break;
case DISK_WRITE: //cmd=1002 write scsi device that in g_sdev[fd]
res=copy_from_user((void *)&cmd_head, (void __user*)arg, sizeof(cmd_head));
if(!res){
printk("kernel write cmd_head struct { %d, 0x%p,%d, %lld } \n",cmd_head.fd,cmd_head.buf,cmd_head.len,cmd_head.offset);
ret_command=do_io( DISK_WRITE, cmd_head.fd, cmd_head.buf, cmd_head.len, cmd_head.offset, cmd_head.seq_num);
}
break;
case DISK_SYNC:
res=copy_from_user((void *)&cmd_head, (void __user*)arg, sizeof(cmd_head));
if(!res){
printk("kernel synchronize cache cmd_head struct { %d, 0x%p,%d, %lld } \n",cmd_head.fd,cmd_head.buf,cmd_head.len,cmd_head.offset);
ret_command=do_sync( DISK_SYNC, cmd_head.fd, cmd_head.seq_num);
}
break;
case DISK_DONE: //cmd=1003 callback by scsi_done ,return to user
res =copy_from_user(&user_buf, (void __user*)arg, sizeof(user_buf));
if(!res){
ret_command = userio_done(user_buf.buf, user_buf.count);
}
break;
default: /* redundant, as cmd was checked against MAXNR */
printk("ioctl cmd=%d not recognize ! ",cmd);
ret_command = ERR_PARAM; //ERR_PARAM=-3
}
return ret_command; // =queuecommand return or done count
}
static int diskio_open(struct inode *inode, struct file *file)
{
/*spin_lock(&spin);
printk("get spinlock\n");
if(chardev_count){
spin_unlock(&spin);
return -EBUSY;
}
chardev_count++;
spin_unlock(&spin);
*/
return 0;
}
static int diskio_release(struct inode *inode, struct file *file)
{
// chardev_count--;
return 0;
}

static const struct file_operations diskio_fops = {
.owner = THIS_MODULE,
.open = diskio_open,
//.ioctl = diskio_ioctl,
.unlocked_ioctl=diskio_ioctl,
.release = diskio_release
};

static int __init diskio_init(void)
{
dev_t dev_id;
int retval;

if (device_major) {
dev_id = MKDEV(device_major, 0);
retval = register_chrdev_region(dev_id, 1, NAME);
} else {
retval = alloc_chrdev_region(&dev_id, 0, 1, NAME);
device_major = MAJOR(dev_id);
}
if(retval){
printk("diskio chrdev region failed\n");
retval = -EBUSY;
goto err_chrdev_region;
}

cdev_init(&diskio_cdev, &diskio_fops);
retval = cdev_add(&diskio_cdev, dev_id, 1);
if (retval) {
printk("diskio cdev add failed\n");
retval = -EBUSY;
goto err_cdev_add;
}

diskio_class = class_create(THIS_MODULE, NAME);
if (IS_ERR(diskio_class)) {
printk("%s: class_create failed\n", NAME);
retval = PTR_ERR(diskio_class);
goto err_class_create;
}

diskio_device = device_create(diskio_class, NULL, dev_id, NULL, "%s", NAME);
if (IS_ERR(diskio_device)) {
printk("%s: class_device_create failed\n", NAME);
retval = PTR_ERR(diskio_device);
goto err_device_create;
}
printk("cdev:board_version:%d\n",chardev_var);
init_MUTEX(&sem);
spin_lock_init(&spin);
init_queue_head(&free_queue_head);
init_queue_head(&done_queue_head);
init_free_queue();
return 0;
err_device_create:
class_destroy(diskio_class);
err_class_create:
cdev_del(&diskio_cdev);
err_cdev_add:
unregister_chrdev_region(dev_id, 1);
err_chrdev_region:
return retval;
}
static void __exit diskio_exit(void)
{
dev_t dev_id = MKDEV(device_major, 0);
device_destroy(diskio_class, dev_id);
class_destroy(diskio_class);
cdev_del(&diskio_cdev);
unregister_chrdev_region(dev_id, 1);
del_queue(free_queue_head, free_list);
del_queue(done_queue_head, done_list);
}
module_init(diskio_init);
module_exit(diskio_exit);
MODULE_DESCRIPTION("disk io test Driver");
MODULE_LICENSE("GPL");
基于BIO  读写硬盘-----------裸盘读写

/*
This block device access with bio.
*/
#include <linux/fs.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/cdev.h>
#include <linux/device.h>
#include <asm/uaccess.h>
#include <linux/semaphore.h>
#include <linux/spinlock.h>
#include <linux/delay.h>
#include <linux/scatterlist.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <scsi/scsi.h>
#include <scsi/scsi_host.h>
#include <scsi/scsi_device.h>
#include <scsi/scsi_transport.h>
#include <scsi/scsi_cmnd.h>
#include <scsi/scsi_dbg.h>
#include <scsi/scsi_eh.h>
#include <linux/list.h>
#define NAME "disk_test_io"
#define DISK_OPEN 1000
#define DISK_READ 1001
#define DISK_WRITE 1002
#define DISK_DONE 1003
#define DISK_SYNC 1004
#define MAX_FREQUEUE_COUNT 1024
#define MAX_DONEQUEUE_COUNT 16
typedef unsigned int uint32;
typedef unsigned long uint64;
typedef unsigned char uint8;
static int chardev_var = 0;
//static int chardev_count = 0;
static struct semaphore sem;
static spinlock_t spin;
static struct cdev diskio_cdev;
static struct class *diskio_class;
static struct device *diskio_device;
static int device_major = 0;

struct block_device *g_bdev[1024]={NULL};
struct deviceid {
int major;
int minor;
int fd;
};

struct iocmd_head{
int fd;
char *buf;
uint32 len; // x*4k
uint32 offset; // x*512 sector
uint64 seq_num;
};
struct cmd_result{
int result;
uint64 seq_num;
struct page* pages[128];
struct list_head free_list;
struct list_head done_list;
};
struct queue_head{
spinlock_t lock;
struct list_head list;
int count;
};
struct queue_head free_queue_head;
struct queue_head done_queue_head;
DECLARE_COMPLETION_ONSTACK(com);
#define del_queue(head, member) ({ \
struct cmd_result *pos,*n; \
if(!list_empty(&head.list)){ \
spin_lock(&head.lock); \
list_for_each_entry_safe(pos, n, &head.list, member) \
kfree(pos); \
head.count=0;\
spin_unlock(&head.lock); \
} \
})
#define get_one_node(head, member) ({ \
struct cmd_result *p = NULL;\
spin_lock(&(head).lock);\
if(list_empty(&head.list)) {\
spin_unlock(&head.lock);\
p = NULL;\
}\
else {\
p = container_of(head.list.next, struct cmd_result, member);\
list_del(head.list.next);\
(head).count--;\
spin_unlock(&(head).lock);}\
p;\
})

#define add_one_node(new, head, member) ({ \
spin_lock(&head.lock);\
head.count++;\
list_add(&(new)->member,&head.list);\
spin_unlock(&head.lock);\
})
void init_queue_head(struct queue_head * q)
{
INIT_LIST_HEAD(&q->list);
q->count = 0;
spin_lock_init(&q->lock);
}
void init_free_queue(void)
{
int i;
struct cmd_result *res = NULL;
for(i = 0; i < MAX_FREQUEUE_COUNT; i++) {
res = kzalloc(sizeof(struct cmd_result), GFP_KERNEL);
if(!res){printk("kzalloc error ! \n");}
add_one_node(res, free_queue_head, free_list);
}
}

int io_open(struct deviceid *dev_id){
int index=0;
dev_t dev=MKDEV(dev_id->major,dev_id->minor);
struct block_device *bdev=open_by_devnum(dev,FMODE_READ|FMODE_WRITE);
if(IS_ERR(bdev)) {
printk("error:failed to open block device [ %d :%d ] !",dev_id->major,dev_id->minor);
return PTR_ERR(bdev);
}
for(index=0;index<1024;index++){ //this bdev already exists
if(g_bdev[index]&&(g_bdev[index]==bdev))
return index;
}
for(index=0;index<1024;index++){ //this bdev is new ,so add to g_sdev
if(g_bdev[index]==NULL){
g_bdev[index]=bdev;
printk("the new bdev=0x%p,\n",bdev);
return index;
}
}
}
void bdev_done(struct bio*bio,int err ){
struct cmd_result *res = (struct cmd_result*)bio->bi_private;
printk("bdev_done !\n");
if (res == NULL)
goto out;
printk("bdev done seq num %ld\n", res->seq_num);
res->result=err;
add_one_node(res, done_queue_head, done_list);
printk("execute generic_make_request return result=%x \n",res->result);
printk("frequeue count = %d, done queue count == %d\n", free_queue_head.count, done_queue_head.count);
out:
complete(&com);
}
static int do_io(int cmd , int fd, char * buf, uint32 len, uint32 offset, uint64 seq_num)
{
int ret=0,nr=0,err=0;
struct bio *bio=NULL;
struct page ** pages=NULL;
int nr_pages=((uint64)buf+len+PAGE_SIZE-1)/PAGE_SIZE-(uint64)buf/PAGE_SIZE;
struct cmd_result *res = get_one_node(free_queue_head, free_list);
if (!res) return -2;
pages = res->pages; //tmp
down_read(¤t->mm->mmap_sem);
ret=get_user_pages(current, current->mm, (unsigned long)buf, nr_pages, 1, 0, pages, NULL);
up_read(¤t->mm->mmap_sem);
if(ret<0) {
printk("get_user_pages failed ! %d ",nr_pages);
return ret;
}
bio=bio_alloc(GFP_KERNEL,nr_pages);
bio->bi_bdev=g_bdev[fd];
//bio->bi_flags=
if(cmd==DISK_READ){
bio->bi_rw |=0;
}else if(cmd==DISK_WRITE){
bio->bi_rw |= 1;
}else {
printk("do_io fail ! cmd=%d ",cmd);
return -1;
}
bio->bi_sector=offset; //start sector
bio->bi_size=0; // residual io count
bio->bi_end_io=bdev_done; // bio done callback func
bio->bi_private=res;
for(nr=0; nr<nr_pages-1; nr++)
{
err=bio_add_page(bio,pages[nr],PAGE_SIZE,0);
if(err<=0) {
printk("bio add page fail !");
return err;
}
}
err=bio_add_page(bio,pages[nr], ((len%PAGE_SIZE)?(len%PAGE_SIZE):PAGE_SIZE), 0);
if(err<=0){
printk("end page add to bio fail !");
return err;
}
generic_make_request(bio);
return 0;
}
static int userio_done(unsigned char* buf, int cnt)
{
int MAX_NUM;
struct cmd_result *result;
MAX_NUM= cnt < MAX_DONEQUEUE_COUNT? cnt: MAX_DONEQUEUE_COUNT;
MAX_NUM= MAX_NUM <done_queue_head.count? MAX_NUM:done_queue_head.count;
int i,ret, count = 0;
struct user_result {
int result;
uint64 seq_num;
//struct scsi_sense_hdr sshr;
};
wait_for_completion(&com);
for(i = 0; i < MAX_NUM; i++) {
result = get_one_node(done_queue_head, done_list);
if(result<=0) {
printk("userio done get node error ! \n");
continue;
}
count++;
ret = copy_to_user((unsigned char*)(buf + i * sizeof(struct user_result)), result, sizeof(struct user_result));
printk("write to user space result result %d, seq_num %ld\n", result->result, result->seq_num);
memset(result, 0, sizeof(struct cmd_result));
add_one_node(result, free_queue_head, free_list);
if (ret < 0)
return -1;
}
return count;
}

int cdev_ioctl (struct inode *inode, struct file *filp,uint32 cmd, uint64 arg)
{
int res=0;
int ret_command=-1;
struct deviceid devid;
struct iocmd_head cmd_head;
struct {
unsigned char* buf;
int count;
} user_buf;
printk(" cdev_ioctl cmd= %d \n",cmd);
switch(cmd) {
case DISK_OPEN: //cmd=0 open to scsi device
res = copy_from_user((void *)&devid, (void __user*)arg, sizeof(devid));
if(!res){ //res =0 copy_from_user success
printk("kernel devid struct { %d,%d} \n",devid.major,devid.minor);
ret_command = io_open(&devid);
if(ret_command >=0){
devid.fd=ret_command;
res = copy_to_user((void __user*)arg, (void *)&devid, sizeof(devid));
printk("== copy_to_user ======res= %d \n",res);
return res; // 没有拷贝的完的字节数
}
}
break;
case DISK_READ: //cmd=1001 read scsi device that in g_sdev[fd]
res=copy_from_user((void *)&cmd_head, (void __user*)arg, sizeof(cmd_head));
if(!res){
printk("kernel read cmd_head struct { %d, 0x%p,%d, %d } \n",cmd_head.fd,cmd_head.buf,cmd_head.len,cmd_head.offset);
ret_command=do_io( DISK_READ, cmd_head.fd, cmd_head.buf, cmd_head.len, cmd_head.offset, cmd_head.seq_num);
}
break;
case DISK_WRITE: //cmd=1002 write scsi device that in g_sdev[fd]
res=copy_from_user((void *)&cmd_head, (void __user*)arg, sizeof(cmd_head));
if(!res){
printk("kernel write cmd_head struct { %d, 0x%p,%d, %d } \n",cmd_head.fd,cmd_head.buf,cmd_head.len,cmd_head.offset);
ret_command=do_io( DISK_WRITE, cmd_head.fd, cmd_head.buf, cmd_head.len, cmd_head.offset, cmd_head.seq_num);
}
break;
/*case DISK_SYNC:
res=copy_from_user((void *)&cmd_head, (void __user*)arg, sizeof(cmd_head));
if(!res){
printk("kernel synchronize cache cmd_head struct { %d, 0x%p,%d, %d } \n",cmd_head.fd,cmd_head.buf,cmd_head.len,cmd_head.offset);
ret_command=do_sync( DISK_SYNC, cmd_head.fd, cmd_head.seq_num);
}
break;
*/
case DISK_DONE: //cmd=1003 callback by scsi_done ,return to user
res =copy_from_user(&user_buf, (void __user*)arg, sizeof(user_buf));
if(!res){
ret_command = userio_done(user_buf.buf, user_buf.count);
}
break;
default: /* redundant, as cmd was checked against MAXNR */
printk("ioctl cmd=%d not recognize ! ",cmd);
ret_command = -3;
}
return ret_command;
}
static int cdev_open(struct inode *inode, struct file *file)
{
return 0;
}
static int cdev_release(struct inode *inode, struct file *file)
{
return 0;
}
static const struct file_operations diskio_fops = {
.owner = THIS_MODULE,
.open = cdev_open,
.ioctl = cdev_ioctl,
.release = cdev_release
};
static int __init diskio_init(void)
{
dev_t dev_id;
int retval;

if (device_major) {
dev_id = MKDEV(device_major, 0);
retval = register_chrdev_region(dev_id, 1, NAME);
} else {
retval = alloc_chrdev_region(&dev_id, 0, 1, NAME);
device_major = MAJOR(dev_id);
}
if(retval){
printk("diskio chrdev region failed\n");
retval = -EBUSY;
goto err_chrdev_region;
}
cdev_init(&diskio_cdev, &diskio_fops);
retval = cdev_add(&diskio_cdev, dev_id, 1);
if (retval) {
printk("diskio cdev add failed\n");
retval = -EBUSY;
goto err_cdev_add;
}
diskio_class = class_create(THIS_MODULE, NAME);
if (IS_ERR(diskio_class)) {
printk("%s: class_create failed\n", NAME);
retval = PTR_ERR(diskio_class);
goto err_class_create;
}
diskio_device = device_create(diskio_class, NULL, dev_id, NULL, "%s", NAME);
if (IS_ERR(diskio_device)) {
printk("%s: class_device_create failed\n", NAME);
retval = PTR_ERR(diskio_device);
goto err_device_create;
}
printk("cdev:board_version:%d\n",chardev_var);
init_MUTEX(&sem);
spin_lock_init(&spin);
init_queue_head(&free_queue_head);
init_queue_head(&done_queue_head);
init_free_queue();
return 0;
err_device_create:
class_destroy(diskio_class);
err_class_create:
cdev_del(&diskio_cdev);
err_cdev_add:
unregister_chrdev_region(dev_id, 1);
err_chrdev_region:
return retval;
}
static void __exit diskio_exit(void)
{
dev_t dev_id = MKDEV(device_major, 0);
device_destroy(diskio_class, dev_id);
class_destroy(diskio_class);
cdev_del(&diskio_cdev);
unregister_chrdev_region(dev_id, 1);
del_queue(free_queue_head, free_list);
del_queue(done_queue_head, done_list);
}
module_init(diskio_init);
module_exit(diskio_exit);
MODULE_DESCRIPTION("disk io test Driver");
MODULE_LICENSE("GPL");
用户态测试程序：(接口基本一样)

#include <stdio.h>
#include <stdlib.h>
#include <malloc.h>
#include <unistd.h>
#define PAGE_SIZE (1<<12)
#define DISK_OPEN 1000
#define DISK_READ 1001
#define DISK_WRITE 1002
#define DISK_DONE 1003
#define DISK_SYNC 1004
typedef unsigned int uint32;
typedef unsigned long uint64;
typedef unsigned char uint8;
struct deviceid {
uint32 hostnum;
uint32 channel;
uint32 id;
uint32 lun;
int fd;
};
struct iocmd_head{
uint32 fd; //sdev[index] --> index = fd
char *buf;
uint32 len; // len=n*512
uint32 offset; //offset is sector num
uint64 seq_num;
};
struct user_result{
int result;
uint64 seq_num;
uint8 response_code;
uint8 sense_key;
uint8 asc;
uint8 ascq;
uint8 byte4;
uint8 byte5;
uint8 byte6;
uint8 additional_length;
};

struct done_input{
struct user_result * buffer; // struct user_result*
int count;
};

int main(int argc,char* argv[])
{
unsigned char *array = valloc(1024);
unsigned char *read_arr = valloc(1024);
int ret,ret_num;
int i=0;
for(i=0;i<1024;i++){
array[i]='g';
}
if(argc!=5){
printf("run this program as ./a.out hostnum channel id lun !\n");
return 0;
}
struct deviceid mydev;
mydev.hostnum=*argv[1]-'0';
mydev.channel=*argv[2]-'0';
mydev.id=*argv[3]-'0';
mydev.lun=*argv[4]-'0';
struct iocmd_head iocmd={ 0 };
iocmd.buf=array;
iocmd.len=512;
iocmd.offset=0; //LBA %512
iocmd.seq_num=1;

printf("user pass devid struct {%d, %d, %d,% d} \n", mydev.hostnum, mydev.channel, mydev.id, mydev.lun);

int fd = open("/dev/disk_test_io", 0070);
//begin to open
int devid = ioctl(fd,DISK_OPEN,&mydev); //open one scsi dev
if(devid==-1 ) {
printf("ioctrl open device error !\n"); //can not find device max devidid 1023
return 0;
}
printf("open this dev with index= %d \n", devid);

//begin to write
iocmd.fd=devid;
printf("array= 0X%x ,iocmd.buf=0X%x\n",array,iocmd.buf);
printf("user pass iocmd struct {%d, 0x%x, %d,% d} \n", iocmd.fd, iocmd.buf, iocmd.len, iocmd.offset);

ret=ioctl(fd, DISK_WRITE, &iocmd);
if(ret!=0){
printf("ioctl write error return %d --0x%x \n",ret,ret);
return 0;
}
printf(" write IOCTRL succes =====ret =%d array=%s \n",ret,array);
sleep(5);
//begin to read
iocmd.buf=read_arr;
ret=ioctl(fd, DISK_READ, &iocmd);
if(ret!=0){
printf(" 2:ioctl read return %d --0x%x \n",ret,ret);
return 0;
}
sleep(5);
printf(" 2:read IOCTRL succes=====ret =%d read_arr=%s \n",ret,read_arr);
//begin ioctrl done
struct user_result result_buf[20];
struct done_input don_buf={
.buffer=result_buf,
.count=10,
};
ret_num = ioctl(fd, DISK_DONE, &don_buf);
if (ret_num) {
int i;
printf("get done io %d\n", ret_num);
for ( i = 0; i < ret_num; i ++)
printf("result %d, seq_num %ld, response_code %d, sense_key %d, asc %d, ascq %d\n", result_buf[i].result, result_buf[i].seq_num,result_buf[i].response_code, result_buf[i].sense_key,result_buf[i].asc,result_buf[i].ascq);
}
ret_num = ioctl(fd, DISK_SYNC, &iocmd);
if (ret_num!=0) {
printf("ioctl sync error ret=%d !\n",ret_num);
return 0;
}
printf("ioctrl sync success !\n");
ret_num = ioctl(fd, DISK_DONE, &don_buf);
if (ret_num) {
int i;
printf("get done io %d\n", ret_num);
for ( i = 0; i < ret_num; i ++)
printf("result %d, seq_num %ld, response_code %d, sense_key %d, asc %d, ascq %d\n", result_buf[i].result, result_buf[i].seq_num,result_buf[i].response_code, result_buf[i].sense_key,result_buf[i].asc,result_buf[i].ascq);
}
}