加载raid驱动的luster内核编译

加载raid驱动的luster内核编译
一、安装流程：

安装OFEB（IB）驱动-->安装lustre内核-->把RAID驱动继承到lustre内核中去-->重新安装OFEB（IB）驱动-->安装lustre工具包-->挂载lustre

二、安装OFEB驱动

1：查看内核版本

[root@localhostopt]# rpm -qa |grep kerne

kernel-2.6.18-194.el5

kernel-devel-2.6.18-194.el5

kernel-headers-2.6.18-194.el5

2：安装OFED (ib驱动)

[root@localhostopt]# ls

libevent-devel-1.4.13-1.x86_64.rpm nfs-utils-lib-devel-1.0.8-7.6.el5.x86_64.rpm OFED-1.5.2-rc2 tcl-devel-8.4.13-4.el5.x86_64.rpm

[root@localhostopt]# rpm -ivh libevent-devel-1.4.13-1.x86_64.rpmnfs-utils-lib-devel-1.0.8-7.6.el5.x86_64.rpm tcl-devel-8.4.13-4.el5.x86_64.rpm

warning:libevent-devel-1.4.13-1.x86_64.rpm: Header V3 DSAsignature: NOKEY, key ID 37017186

Preparing... ########################################### [100%]

1:tcl-devel ###########################################[ 33%]

2:libevent-devel ########################################### [ 67%]

3:nfs-utils-lib-devel ###########################################[100%]

[root@localhostopt]# cd OFED-1.5.2-rc2/

[root@localhostOFED-1.5.2-rc2]# ls

BUILD_ID docs in1.sh install.pl LICENSE ofed.conf ofed.conf.ok README.txt RPMS SRPMS un0.sh uninstall.exp uninstall.sh

[root@localhostOFED-1.5.2-rc2]# ./un0.sh

[root@localhostOFED-1.5.2-rc2]# ./in1.sh

三、安装lustre内核

[root@localhostlustre1.8.5]# rpm -ivh --forcekernel-2.6.18-194.17.1.0.1.el5_lustre.1.8.5.x86_64.rpmkernel-headers-2.6.18-194.17.1.0.1.el5_lustre.1.8.5.x86_64.rpmkernel-devel-2.6.18-194.17.1.el5_lustre.1.8.5.x86_64.rpm

Preparing... ###########################################[100%]

1:kernel-devel ########################################### [ 33%]

2:kernel ########################################### [ 67%]

WARNING:No module megasr found for kernel 2.6.18-194.17.1.0.1.el5_lustre.1.8.5,continuing anyway

WARNING:No module megasr found for kernel 2.6.18-194.17.1.0.1.el5_lustre.1.8.5,continuing anyway

WARNING:No module megasr found for kernel 2.6.18-194.17.1.0.1.el5_lustre.1.8.5,continuing anyway

3:kernel-headers ########################################### [100%]

[root@localhostlustre1.8.5]# rpm -qa |grep lustre

kernel-devel-2.6.18-194.17.1.el5_lustre.1.8.5

kernel-headers-2.6.18-194.17.1.0.1.el5_lustre.1.8.5

kernel-2.6.18-194.17.1.0.1.el5_lustre.1.8.5

四、把RAID驱动继承到lustre内核中去

1.编译RAID驱动

[root@localhostopt]# tar zxvf 13.10.0708.2009_Linux_SWR_Driver_R5_Univ_for_2.6.30_x86_64.tar.gz

[root@localhostshim]# patch -p0 < megasr_open_source.patch

patchingfile open_source/osl/lin/linux_osl.c

patchingfile open_source/osl/lin/linux_osl.h

此步骤需要执行两次，原因不知道

[root@localhostshim]# cd

library/ megasr_open_source.patch open_source/

[root@localhostshim]# cd library/rhel5-64/

[root@localhostrhel5-64]# make -C /usr/src/kernels/2.6.18-194.17.1.el5_lustre.1.8.5-x86_64/M=$(pwd)

[root@localhostrhel5-64]# ls

built-in.o megasr.ko megasr.mod.c megasr.o Module.symvers

Makefile megasr-lib.a megasr.mod.o Module.markers

（编译成功，会生成megasr.ko驱动模块）

2：把RAID驱动集成到lustre内核中

[root@localhost~]#mkdir initrd

[root@localhost~]# cd initrd/

[root@localhostinitrd]# cp /boot/initrd-2.6.18-194.17.1.0.1.el5_lustre.1.8.5.img.

[root@localhostinitrd]# mv initrd-2.6.18-194.17.1.0.1.el5_lustre.1.8.5.imginitrd-2.6.

.18-194.17.1.0.1.el5_lustre.1.8.5.img.gz

[root@localhostinitrd]# gunzip initrd-2.6.18-194.17.1.0.1.el5_lustre.1.8.5.img.gz

[root@localhostinitrd]# cpio -div < initrd-2.6.18-194.17.1.0.1.el5_lustre.1.8.5.img

[root@localhostinitrd]# ls

bin etc initrd-2.6.18-194.17.1.0.1.el5_lustre.1.8.5.img proc sys

dev init lib sbin sysroot

[root@localhostinitrd]# cp /opt/13.10.0708.2009_Linux_SWR_Driver_R5_Univ_for_2.6.30_x8

86_64/shim/library/rhel5-64/megasr.kolib

[root@localhostlib]# chmod 600 megasr.ko

[root@localhostlib]# modinfo megasr.ko |grep vermagic

vermagic: 2.6.18-194.17.1.el5_lustre.1.8.5SMP mod_unload gcc-4.1 （查看是否为lustre内核编译上驱动）

[root@localhostinitrd]# vi init

"init"

"init"93L, 2469C#!/bin/nash

mount -tproc /proc /proc

setquiet

echoMounting proc filesystem

echoMounting sysfs filesystem

mount -tsysfs /sys /sys

echoCreating /dev

mount -omode=0755 -t tmpfs /dev /dev

mkdir/dev/pts

mount -tdevpts -o gid=5,mode=620 /dev/pts /dev/pts

mkdir/dev/shm

mkdir/dev/mapper

echoCreating initial device nodes

mknod/dev/null c 1 3

mknod/dev/zero c 1 5

mknod/dev/urandom c 1 9

mknod/dev/systty c 4 0

mknod/dev/tty c 5 0

mknod/dev/console c 5 1

mknod/dev/ptmx c 5 2

mknod/dev/rtc c 10 135

mknod/dev/tty0 c 4 0

mknod/dev/tty1 c 4 1

mknod/dev/tty2 c 4 2

mknod/dev/tty3 c 4 3

mknod/dev/tty4 c 4 4

mknod/dev/tty5 c 4 5

mknod/dev/tty6 c 4 6

mknod/dev/tty7 c 4 7

mknod/dev/tty8 c 4 8

mknod/dev/tty9 c 4 9

mknod/dev/tty10 c 4 10

mknod/dev/tty11 c 4 11

mknod/dev/tty12 c 4 12

mknod/dev/ttyS0 c 4 64

mknod/dev/ttyS1 c 4 65

mknod/dev/ttyS2 c 4 66

mknod/dev/ttyS3 c 4 67

echoSetting up hotplug.

hotplug

echoCreating block device nodes.

mkblkdevs

echo"Loading ehci-hcd.ko module"

insmod/lib/ehci-hcd.ko

echo"Loading ohci-hcd.ko module"

insmod/lib/ohci-hcd.ko

echo"Loading uhci-hcd.ko module"

insmod/lib/uhci-hcd.ko

mount -tusbfs /proc/bus/usb /proc/bus/usb

echo"Loading jbd.ko module"

insmod/lib/jbd.ko

echo"Loading ext3.ko module"

insmod/lib/ext3.ko

echo"Loading shpchp.ko module"

insmod/lib/shpchp.ko

echo"Loading scsi_mod.ko module"

insmod/lib/scsi_mod.ko

echo"Loading sd_mod.ko module"

insmod/lib/sd_mod.ko

echo"Loading libata.ko module"

insmod/lib/libata.ko

echo"Loading ata_piix.ko module"

insmod/lib/ata_piix.ko

echo"Loading scsi_transport_fc.ko module"

insmod/lib/scsi_transport_fc.ko

echo"Loading lpfc.ko module"

insmod/lib/lpfc.ko

echo"Loading dm-mem-cache.ko module"

insmod/lib/scsi_transport_fc.ko-- INSERT --- INSERT --echo "Loading megasr.komodule"-- INSERT --insmod /lib/megasr.komkblkdevs

echo"Loading ehci-hcd.ko module"

insmod/lib/ehci-hcd.ko

echo"Loading ohci-hcd.ko module"

insmod/lib/ohci-hcd.ko

echo"Loading uhci-hcd.ko module"

insmod/lib/uhci-hcd.ko

mount -tusbfs /proc/bus/usb /proc/bus/usb

echo"Loading jbd.ko module"

insmod/lib/jbd.ko

echo"Loading ext3.ko module"

insmod/lib/ext3.ko

echo"Loading shpchp.ko module"

insmod/lib/shpchp.ko

echo"Loading scsi_mod.ko module"

insmod/lib/scsi_mod.ko

echo "Loading megasr.ko module"------------------------（在此位置添加红色字体）

insmod /lib/megasr.ko

echo"Loading sd_mod.ko module"

insmod/lib/sd_mod.ko

echo"Loading libata.ko module"

insmod/lib/libata.ko

echo"Loading ata_piix.ko module"

insmod/lib/ata_piix.ko

echo"Loading scsi_transport_fc.ko module"

insmod/lib/scsi_transport_fc.ko

echo"Loading lpfc.ko module"

insmod/lib/lpfc.ko

echo"Loading dm-mem-cache.ko module":wq

"init"

"init"95L, 2523C written

将内核重新压缩，并修改grub配置文件

[root@localhostinitrd]# rm -fr initrd-2.6.18-194.17.1.0.1.el5_lustre.1.8.5.img

[root@localhostinitrd]# find ./ | cpio -H newc -o > /boot/initrd-2.6.18-194.17.1.0.1.el5_lustre.1.8.5.img.new

[root@localhostinitrd]# cd /boot/

[root@localhostboot]# gzip initrd-2.6.18-194.17.1.0.1.el5_lustre.1.8.5.img.new

[root@localhostboot]# mv initrd-2.6.18-194.17.1.0.1.el5_lustre.1.8.5.img.new.gzinitrd-2.6.18-194.17.1.0.1.el5_lustre.1.8.5.img.new

[root@localhostboot]# vi grub/grub.conf

"grub/grub.conf"

"grub/grub.conf"21L, 819C# grub.conf generated by anaconda

#

# Notethat you do not have to rerun grub after making changes to this file

#NOTICE: You have a /boot partition. This means that

#allkernel and initrd paths are relative to /boot/, eg.

#root(hd0,0)

#kernel/vmlinuz-version ro root=/dev/sda3

#initrd/initrd-version.img

#boot=/dev/sda

default=0

timeout=5

splashimage=(hd0,0)/grub/splash.xpm.gz

hiddenmenu

title RedHat Enterprise Linux Server (2.6.18-194.17.1.0.1.el5_lustre.1.8.5)root(hd0,0)kernel
/vmlinuz-2.6.18-194.17.1.0.1.el5_lustre.1.8.5ro root=LABEL=/ quietinitrd/initrd-2.6.18-194.17.1.0.1.el5_lustre.1.8.5.img.new (用新内核启动系统)

title RedHat Enterprise Linux Server (2.6.18-194.el5)root(hd0,0)kernel /vmlinuz-2.6.18-194.el5 ro root=LABEL=/ quietinitrd /initrd-2.6.18-194.el5.img

~ ~ initrd/initrd-2.6.18-194.17.1.0.1.el5_lustre.1.8.5.im--INSERT --.new:wq

"grub/grub.conf"

"grub/grub.conf"21L, 823C written

五、重新安装OFED驱动

六、安装lustre工具包

1.自己生成rpm安装包

[root@login3tmp]# tar -zxvf lustre-1.8.5.tar.gz 生成 lustre-1.8.5 文件夹.

[root@login3tmp]# cd lustre-1.8.5

[root@login3lustre-1.8.5]#./configure--with-linux=/usr/src/kernel/<LUSTRE-KERNEL> --with=/usr/src/ofa_kernel

[root@login3lustre-1.8.5]#makerpms 会在/usr/src/redhat/RPMS/x86_64目录下生成需要安装的rpm文件

[root@login3 x86_64]# pwd

/usr/src/redhat/RPMS/x86_64

[root@login3 x86_64]# ls

lustre-1.8.5-2.6.18_194.17.1.el5_lustre.1.8.5_201204080649.x86_64.rpm

lustre-debuginfo-1.8.5-2.6.18_194.17.1.el5_lustre.1.8.5_201204080649.x86_64.rpm

lustre-ldiskfs-3.1.4-2.6.18_194.17.1.el5_lustre.1.8.5_201204080650.x86_64.rpm

lustre-ldiskfs-debuginfo-3.1.4-2.6.18_194.17.1.el5_lustre.1.8.5_201204080650.x86_64.rpm

lustre-modules-1.8.5-2.6.18_194.17.1.el5_lustre.1.8.5_201204080649.x86_64.rpm

lustre-source-1.8.5-2.6.18_194.17.1.el5_lustre.1.8.5_201204080649.x86_64.rpm

lustre-tests-1.8.5-2.6.18_194.17.1.el5_lustre.1.8.5_201204080649.x86_64.rpm

2.安装自己生成的rpm包

[root@login3 x86_64]# rpm -Uvh --force*.rpm. 注意如果此步安装不成功，可能是需要 expect软件包。可以到网上去下，或者是从系统的镜像文件系统中提取，为了和系统兼容，个人建议还是从镜像文件中提取比较好。

六、挂载lustre系统

1、将options lnet networks=o2ib 加到/etc/modprob.conf

[root@login3 ~]# vim /etc/modprobe.conf

alias eth0 igb

alias eth1 igb

alias scsi_hostadapter megasr

alias scsi_hostadapter1 ata_piix

alias scsi_hostadapter2 usb-storage

options lnetnetworks=o2ib



2、启动IB网模块

[root@login3 ~]# modprobe lnet

[root@login3 ~]# lctl network up

LNET configured

3、这步不成功有可能是：

1、之前source过 /etc/rc.local 文件，解决办法如下：

lctlnetwork down;

lctlnetwork unconfigure;

lustre_rmmod;

modprobelnet;

lctlnetwork up;

2、重启机器

4、查看是否启动成功

[root@login3 ~]# lctl list_nids

10.0.5.3@o2ib

5、启动完毕，如果成功，则挂载即可，挂载方法是编辑/etc/rc.local

[root@login3 ~]# vi /etc/rc.local

#!/bin/sh

#

# This script will be executed *after* allthe other init scripts.

# You can put your own initialization stuffin here if you don't

# want to do the full Sys V style initstuff.

touch /var/lock/subsys/local

##for lustre##

sleep 20

modprobe lnet

lctl network up

sleep 50

mount.lustre 10.0.4.1@o2ib:10.0.4.2@o2ib:/jdhpc/lustre/

[root@login3 ~]# source /etc/rc.local

6、用mount命令查看lustre是否被挂载

[root@login3 ~]# mount

/dev/mapper/VolGroup00-LogVol01 on / typeext3 (rw)

proc on /proc type proc (rw)

sysfs on /sys type sysfs (rw)

devpts on /dev/pts type devpts (rw,gid=5,mode=620)

/dev/sda1 on /boot type ext3 (rw)

tmpfs on /dev/shm type tmpfs (rw)

none on /proc/sys/fs/binfmt_misc typebinfmt_misc (rw)

sunrpc on /var/lib/nfs/rpc_pipefs typerpc_pipefs (rw)

10.0.4.1@o2ib:10.0.4.2@o2ib:/jdhpc on /lustre type lustre (0)

可以看到lustre已经被挂载成功。