cephfs kernel client debug

Posted on | In | |

获取源码

1
2
# uname -a
Linux Server 3.10.0-327.el7.x86_64 #1 SMP Thu Nov 19 22:10:57 UTC 2015 x86_64 x86_64 x86_64 GNU/Linux

从网上搜索下载Linux对应版本的源码，这里的源码版本为：3.10.0-327

查看cephfs client端模块

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
# lsmod | grep ceph
ceph                  202130  1
libceph               189287  2 rbd,ceph

# modinfo ceph
filename:       /lib/modules/3.10.0-327.el7.x86_64/kernel/fs/ceph/ceph.ko
license:        GPL
description:    Ceph filesystem for Linux
author:         Patience Warnick <patience@newdream.net>
author:         Yehuda Sadeh <yehuda@hq.newdream.net>
author:         Sage Weil <sage@newdream.net>
alias:          fs-ceph
rhelversion:    7.2
srcversion:     268CE83A90FA60A7654BE27
depends:        libceph
intree:         Y
vermagic:       3.10.0-327.el7.x86_64 SMP mod_unload modversions
signer:         CentOS Linux kernel signing key
sig_key:        79:AD:88:6A:11:3C:A0:22:35:26:33:6C:0F:82:5B:8A:94:29:6A:B3
sig_hashalgo:   sha25

# modinfo libceph
filename:       /lib/modules/3.10.0-327.el7.x86_64/kernel/net/ceph/libceph.ko
license:        GPL
description:    Ceph filesystem for Linux
author:         Patience Warnick <patience@newdream.net>
author:         Yehuda Sadeh <yehuda@hq.newdream.net>
author:         Sage Weil <sage@newdream.net>
rhelversion:    7.2
srcversion:     BA3AB21E57822D9AC6B9463
depends:        libcrc32c,dns_resolver
intree:         Y
vermagic:       3.10.0-327.el7.x86_64 SMP mod_unload modversions
signer:         CentOS Linux kernel signing key
sig_key:        79:AD:88:6A:11:3C:A0:22:35:26:33:6C:0F:82:5B:8A:94:29:6A:B3
sig_hashalgo:   sha256

修改cephfs源码

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
到kernel源码目录
# vim include/linux/ceph/ceph_debug.h
...
# if defined(DEBUG) || defined(CONFIG_DYNAMIC_DEBUG)
/* 删除这块代码
extern const char *ceph_file_part(const char *s, int len);
#  define dout(fmt, ...)                        \
    pr_debug("%.*s %12.12s:%-4d : " fmt,                \
         8 - (int)sizeof(KBUILD_MODNAME), "    ",       \
         ceph_file_part(__FILE__, sizeof(__FILE__)),        \
         __LINE__, ##__VA_ARGS__)
# else
*/
/* faux printk call just to see any compiler warnings. */
#  define dout(fmt, ...)    do {                \
        if (1)                      \                   // if(0) 修改为 if(1)
            printk(KERN_DEBUG fmt, ##__VA_ARGS__);  \
    } while (0)
# endif

#else
...

结合自己版本里的源码，随意修改，使dout()函数能正常打印log就ok；

1
2
3
4
5
6
[root@server linux-3.10.0-327.el7]# cp /boot/config-3.10.0-327.el7.x86_64 ./
[root@server linux-3.10.0-327.el7]# mv config-3.10.0-327.el7.x86_64 .config
[root@server linux-3.10.0-327.el7]# vim .config
...
CONFIG_CEPH_LIB_PRETTYDEBUG=y		// 打开CEPH的相关DEBUG配置
...

编译并替换cephfs的模块

编译cephfs使用的kernel module

1
2
3
4
5
6
7
[root@server linux-3.10.0-327.el7]# make
[root@server linux-3.10.0-327.el7]# ll fs/ceph/ceph.ko
-rw-r--r-- 1 root root 4901829 Jun  7 09:39 fs/ceph/ceph.ko
[root@server linux-3.10.0-327.el7]# ll net/ceph/libceph.ko
-rw-r--r-- 1 root root 4099969 Jun  7 09:40 net/ceph/libceph.ko
[root@server linux-3.10.0-327.el7]# ll drivers/block/rbd.ko
-rw-r--r-- 1 root root 966206 Jun  7 09:37 drivers/block/rbd.ko

替换cephfs的这几个模块

1
2
3
4
5
6
7
8
9
10
11
12
13
14
[root@server linux-3.10.0-327.el7]# lsmod | grep rbd
rbd                    58242  0
libceph               189287  2 rbd,ceph    // rbd使用libceph module，替换libceph的话，先要删除rbd module

[root@server linux-3.10.0-327.el7]# rmmod rbd
[root@server linux-3.10.0-327.el7]# rmmod ceph
[root@server linux-3.10.0-327.el7]# rmmod libceph

[root@server linux-3.10.0-327.el7]# cp net/ceph/libceph.ko /lib/modules/3.10.0-327.el7.x86_64/kernel/net/ceph/libceph.ko
[root@server linux-3.10.0-327.el7]# cp fs/ceph/ceph.ko /lib/modules/3.10.0-327.el7.x86_64/kernel/fs/ceph/ceph.ko
[root@server linux-3.10.0-327.el7]# cp drivers/block/rbd.ko /lib/modules/3.10.0-327.el7.x86_64/kernel/drivers/block/rbd.ko

[root@server linux-3.10.0-327.el7]# modprobe ceph
[root@server linux-3.10.0-327.el7]# modprobe rbd

并不是每次修改cephfs的代码，都需要编译整个kernel的，后续仅仅需要编译对应的module即可

1
[root@server linux-3.10.0-327.el7]# make M=fs/ceph

测试查看cephfs debug log

mount cephfs并触发写操作

1
2
3
# mount -t ceph 10.10.1.1:6789:/ /mnt/
# cd /mnt
# dd if=/dev/zero of=tstfile bs=4M count=1 oflag=direct

检查cephfs的kernel log

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
# dmesg -c > /dev/null
# dmesg -c > ~/cephfs.log
# vim ~/cephfs.log
...
[3034497.087430] aio_write ffff8801271b0e48 10000002393.fffffffffffffffe 0~4194304 got cap refs on Fwb
[3034497.087432] sync_direct_write on file ffff882013dddf00 0~4194304
[3034497.087449] ceph_msg_new ffff88135e1ec000 front 512
[3034497.087450] ceph_msg_new ffff88135e1ec0f0 front 277
[3034497.087451] mapping 0~4194304  osize 4194304 fl_su 4194304
[3034497.087452] osize 4194304 / su 4194304 = su_per_object 1
[3034497.087454] off 0 / su 4194304 = bl 0
[3034497.087455] objset 0 * sc 1 = ono 0
[3034497.087455]  obj extent 0~4194304
[3034497.087457] calc_layout objnum=0 0~4194304
[3034497.087576] oid '10000002393.00000000' len 20
[3034497.087578] build_request msg_size was 197
[3034497.087580] __register_request ffff88111d798000 tid 8
[3034497.087585] ceph_osdc_get_request ffff88111d798000 (was 1)
...

配置kernel log文件

若不想每次从dmesg里获取kernel debug log，在centos里可以配置到指定文件

1
2
3
4
5
6
# vim /etc/rsyslog.conf
...
kern.*                                                 /var/log/kern.log   // 配置kernel log文件
...

# service rsyslog restart		// 重启rsyslog服务

之后就可以在/var/log/kern.log里查看cephfs的debug log了

支持原创