I'm getting a kernel oops indicating a null pointer dereference in
xfs_trans_mod_dquot:
crash> bt
PID: 484420 TASK: ffff88003914e580 CPU: 0 COMMAND: "webalizer"
#0 [ffff8801657ab758] machine_kexec at ffffffff8105249b
#1 [ffff8801657ab7b8] crash_kexec at ffffffff811034f2
#2 [ffff8801657ab888] oops_end at ffffffff8163d9e8
#3 [ffff8801657ab8b0] no_context at ffffffff8162e64b
#4 [ffff8801657ab900] __bad_area_nosemaphore at ffffffff8162e6e1
#5 [ffff8801657ab950] bad_area at ffffffff8162ea24
#6 [ffff8801657ab978] __do_page_fault at ffffffff8164091c
#7 [ffff8801657ab9d8] do_page_fault at ffffffff81640993
#8 [ffff8801657aba00] page_fault at ffffffff8163cb88
[exception RIP: xfs_trans_mod_dquot+56]
RIP: ffffffffa0305768 RSP: ffff8801657abab0 RFLAGS: 00010246
RAX: 0000000000000000 RBX: ffff88019f3842b8 RCX: 000000000000002a
RDX: 0000000000010000 RSI: ffff8800c235df58 RDI: ffff88019f3842f8
RBP: ffff8801657abad8 R8: ffff8800c235e088 R9: 0000000000000000
R10: 000000000000002a R11: ffff880413c93800 R12: 0000000000010000
R13: ffff8800c235df58 R14: 000000000000002a R15: ffff88019f3842f8
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
#9 [ffff8801657abae0] xfs_trans_dqresv at ffffffffa0305c47 [xfs]
#10 [ffff8801657abb50] xfs_trans_reserve_quota_bydquots at ffffffffa03062ee
[xfs]
#11 [ffff8801657abb90] xfs_create at ffffffffa02e76f2 [xfs]
#12 [ffff8801657abc50] xfs_vn_mknod at ffffffffa02e3e99 [xfs]
#13 [ffff8801657abcb8] xfs_vn_create at ffffffffa02e4043 [xfs]
#14 [ffff8801657abcc8] vfs_create at ffffffff81207b7c
#15 [ffff8801657abd00] do_last at ffffffff812096ed
#16 [ffff8801657abda8] path_openat at ffffffff8120ab12
#17 [ffff8801657abe48] do_filp_open at ffffffff8120d82b
#18 [ffff8801657abf18] do_sys_open at ffffffff811fa3a3
#19 [ffff8801657abf70] sys_open at ffffffff811fa4be
#20 [ffff8801657abf80] system_call_fastpath at ffffffff81645189
RIP: 00007f1067c3c850 RSP: 00007ffd6cb59bd8 RFLAGS: 00000246
RAX: 0000000000000002 RBX: ffffffff81645189 RCX: ffffffffffffffff
RDX: 00000000000001b6 RSI: 0000000000000241 RDI: 00007ffd6cb58ad0
RBP: 00007ffd6cb58a60 R8: 000000000041cf15 R9: 0000000000000240
R10: 0000000000000024 R11: 0000000000000246 R12: ffffffff811fa4be
R13: ffff8801657abf78 R14: 0000000000000023 R15: 0000000000000001
ORIG_RAX: 0000000000000002 CS: 0033 SS: 002b
I'm using OpenVZ kernel rh7-3.10.0-327.10.1.vz7.12.14 located at
https://github.com/OpenVZ/vzkernel/releases/tag/rh7-3.10.0-327.10.1.vz7.12.14
Here is some info from coredump (hope it will be useful):
crash> bt -f
..
#8 [ffff8801657aba00] page_fault at ffffffff8163cb88
[exception RIP: xfs_trans_mod_dquot+56]
RIP: ffffffffa0305768 RSP: ffff8801657abab0 RFLAGS: 00010246
RAX: 0000000000000000 RBX: ffff88019f3842b8 RCX: 000000000000002a
RDX: 0000000000010000 RSI: ffff8800c235df58 RDI: ffff88019f3842f8
RBP: ffff8801657abad8 R8: ffff8800c235e088 R9: 0000000000000000
R10: 000000000000002a R11: ffff880413c93800 R12: 0000000000010000
R13: ffff8800c235df58 R14: 000000000000002a R15: ffff88019f3842f8
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
ffff8801657aba08: ffff88019f3842f8 000000000000002a
ffff8801657aba18: ffff8800c235df58 0000000000010000
ffff8801657aba28: ffff8801657abad8 ffff88019f3842b8
ffff8801657aba38: ffff880413c93800 000000000000002a
ffff8801657aba48: 0000000000000000 ffff8800c235e088
ffff8801657aba58: 0000000000000000 000000000000002a
ffff8801657aba68: 0000000000010000 ffff8800c235df58
ffff8801657aba78: ffff88019f3842f8 ffffffffffffffff
ffff8801657aba88: ffffffffa0305768 0000000000000010
ffff8801657aba98: 0000000000010246 ffff8801657abab0
ffff8801657abaa8: 0000000000000018 ffff8800c235df58
ffff8801657abab8: 0000000000000001 0000000000010000
ffff8801657abac8: ffff8800c235e0c8 ffff88019f3842b8
ffff8801657abad8: ffff8801657abb48 ffffffffa0305c47
#9 [ffff8801657abae0] xfs_trans_dqresv at ffffffffa0305c47 [xfs]
ffff8801657abae8: ffff880411cf6100 0000000000000000
ffff8801657abaf8: 0000000000000000 ffff8801657abb30
ffff8801657abb08: 000000000000002a ffff880413c93800
ffff8801657abb18: ffff880415decc00 ffff880413c93800
ffff8801657abb28: ffff88019f3842b8 ffff88041546c000
ffff8801657abb38: 000000000000002a 0000000000000000
ffff8801657abb48: ffff8801657abb88 ffffffffa03062ee
#10 [ffff8801657abb50] xfs_trans_reserve_quota_bydquots at ffffffffa03062ee
[xfs]
ffff8801657abb58: ffff8800c235df58 ffff8801a642dd80
ffff8801657abb68: ffff880413c93800 ffff8801657abc70
ffff8801657abb78: 000000000000000a 0000000000000000
ffff8801657abb88: ffff8801657abc48 ffffffffa02e76f2
#11 [ffff8801657abb90] xfs_create at ffffffffa02e76f2 [xfs]
ffff8801657abb98: 0000000000000001 ffff880100010000
ffff8801657abba8: ffff8801657abbd8 ffff8801657abc68
ffff8801657abbb8: ffff880413c93b48 0000000000000000
ffff8801657abbc8: 0000002a000081a4 ffffffff81213ec2
ffff8801657abbd8: 0000000000000000 ffff88019f3842b8
ffff8801657abbe8: ffffffffffffffff ffff88041546c000
ffff8801657abbf8: 0000000000000000 ffff8800c235df58
ffff8801657abc08: 0000000000000000 0000000000000000
ffff8801657abc18: 0000000051c3d868 ffff8801610e1680
ffff8801657abc28: 0000000000000000 ffff8801a642df38
ffff8801657abc38: 00000000000081a4 00000000000081b6
ffff8801657abc48: ffff8801657abcb0 ffffffffa02e3e99
#12 [ffff8801657abc50] xfs_vn_mknod at ffffffffa02e3e99 [xfs]
ffff8801657abc58: ffffffff81206900 ffff880100000000
ffff8801657abc68: 0000000000000000 ffff8801610e16b8
ffff8801657abc78: 0000000100000015 0000000051c3d868
ffff8801657abc88: 0000000000000000 ffff8801a642df38
ffff8801657abc98: ffff8801610e1680 00000000000081b6
ffff8801657abca8: 0000000000000000 ffff8801657abcc0
ffff8801657abcb8: ffffffffa02e4043
#13 [ffff8801657abcb8] xfs_vn_create at ffffffffa02e4043 [xfs]
ffff8801657abcc0: ffff8801657abcf8 ffffffff81207b7c
#14 [ffff8801657abcc8] vfs_create at ffffffff81207b7c
ffff8801657abcd0: ffff8801657abf28 0000000000008241
ffff8801657abce0: ffff8801657abe50 ffff8801013255c0
ffff8801657abcf0: ffff8801610e1680 ffff8801657abda0
ffff8801657abd00: ffffffff812096ed
#15 [ffff8801657abd00] do_last at ffffffff812096ed
ffff8801657abd08: ffffea000d264240 0000000000000000
ffff8801657abd18: ffff8801657abd68 ffff8801657abd90
ffff8801657abd28: ffff8801a642df38 ffff8803f5e18000
ffff8801657abd38: ffff88001f9cd000 ffff8801657abde4
ffff8801657abd48: ffff8801013255c0 0100000100000022
ffff8801657abd58: ffff8801657abdf0 00ff88001f9cd000
ffff8801657abd68: ffff8801d7d1a000 0000000051c3d868
ffff8801657abd78: ffff8801657abe50 ffff88001f9cd000
ffff8801657abd88: ffff8803f5e18000 ffff8801657abf28
ffff8801657abd98: ffff88003914e580 ffff8801657abe40
ffff8801657abda8: ffffffff8120ab12
#16 [ffff8801657abda8] path_openat at ffffffff8120ab12
ffff8801657abdb0: ffff8801657abe80 ffffffff8120d732
ffff8801657abdc0: ffff880413bcb720 ffff8801610e1680
ffff8801657abdd0: 0000001553594ce5 00000041f5e18020
ffff8801657abde0: 0000000100000000 0000000000000000
ffff8801657abdf0: ffff8801a642df38 0000000200000000
ffff8801657abe00: 0000000000000000 00007f1068e7c000
ffff8801657abe10: 0000000051c3d868 00000000ffffff9c
ffff8801657abe20: ffff8803f5e18000 ffff8801657abf28
ffff8801657abe30: 0000000000000001 0000000000000023
ffff8801657abe40: ffff8801657abf10 ffffffff8120d82b
#17 [ffff8801657abe48] do_filp_open at ffffffff8120d82b
ffff8801657abe50: ffff880413bcb720 ffff8801013255c0
ffff8801657abe60: 0000001553594ce5 ffff8803f5e18020
ffff8801657abe70: 0000000000000000 00007ffd6cb58ad0
ffff8801657abe80: ffff8801a642df38 0000000200000301
ffff8801657abe90: 0000000000000000 0000000000000001
ffff8801657abea0: 00007ffd6cb58ad0 0000000000000000
ffff8801657abeb0: 0000000000000000 ffff8801657abf00
ffff8801657abec0: ffffffff8121a867 ffff880413ca91c0
ffff8801657abed0: 0000ffff00008241 0000000000000001
ffff8801657abee0: 0000000000008241 0000000051c3d868
ffff8801657abef0: 0000000000000001 0000000000000005
ffff8801657abf00: 00000000ffffff9c ffff8803f5e18000
ffff8801657abf10: ffff8801657abf68 ffffffff811fa3a3
#18 [ffff8801657abf18] do_sys_open at ffffffff811fa3a3
ffff8801657abf20: ffff880286c5b648 ffff81b600008241
ffff8801657abf30: 0000030000000022 0000000051c3d868
ffff8801657abf40: 000000000041cf0f 0000000001aa8300
ffff8801657abf50: 0000000000000004 0000000000000001
ffff8801657abf60: 0000000000000023 ffff8801657abf78
ffff8801657abf70: ffffffff811fa4be
#19 [ffff8801657abf70] sys_open at ffffffff811fa4be
ffff8801657abf78: 00007ffd6cb58a60 ffffffff81645189
#20 [ffff8801657abf80] system_call_fastpath at ffffffff81645189
RIP: 00007f1067c3c850 RSP: 00007ffd6cb59bd8 RFLAGS: 00000246
RAX: 0000000000000002 RBX: ffffffff81645189 RCX: ffffffffffffffff
RDX: 00000000000001b6 RSI: 0000000000000241 RDI: 00007ffd6cb58ad0
RBP: 00007ffd6cb58a60 R8: 000000000041cf15 R9: 0000000000000240
R10: 0000000000000024 R11: 0000000000000246 R12: ffffffff811fa4be
R13: ffff8801657abf78 R14: 0000000000000023 R15: 0000000000000001
ORIG_RAX: 0000000000000002 CS: 0033 SS: 002b
crash> bt -F
...
#8 [ffff8801657aba00] page_fault at ffffffff8163cb88
[exception RIP: xfs_trans_mod_dquot+56]
RIP: ffffffffa0305768 RSP: ffff8801657abab0 RFLAGS: 00010246
RAX: 0000000000000000 RBX: ffff88019f3842b8 RCX: 000000000000002a
RDX: 0000000000010000 RSI: ffff8800c235df58 RDI: ffff88019f3842f8
RBP: ffff8801657abad8 R8: ffff8800c235e088 R9: 0000000000000000
R10: 000000000000002a R11: ffff880413c93800 R12: 0000000000010000
R13: ffff8800c235df58 R14: 000000000000002a R15: ffff88019f3842f8
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
ffff8801657aba08: [cfq_queue] 000000000000002a
ffff8801657aba18: ffff8800c235df58 0000000000010000
ffff8801657aba28: ffff8801657abad8 [cfq_queue]
ffff8801657aba38: ffff880413c93800 000000000000002a
ffff8801657aba48: 0000000000000000 ffff8800c235e088
ffff8801657aba58: 0000000000000000 000000000000002a
ffff8801657aba68: 0000000000010000 ffff8800c235df58
ffff8801657aba78: [cfq_queue] ffffffffffffffff
ffff8801657aba88: xfs_trans_mod_dquot+56 0000000000000010
ffff8801657aba98: 0000000000010246 ffff8801657abab0
ffff8801657abaa8: 0000000000000018 ffff8800c235df58
ffff8801657abab8: 0000000000000001 0000000000010000
ffff8801657abac8: ffff8800c235e0c8 [cfq_queue]
ffff8801657abad8: ffff8801657abb48 xfs_trans_dqresv+647
#9 [ffff8801657abae0] xfs_trans_dqresv at ffffffffa0305c47 [xfs]
ffff8801657abae8: [kmem_cache] 0000000000000000
ffff8801657abaf8: 0000000000000000 ffff8801657abb30
ffff8801657abb08: 000000000000002a ffff880413c93800
ffff8801657abb18: [kmalloc-512] ffff880413c93800
ffff8801657abb28: [cfq_queue] [xfs_dquot]
ffff8801657abb38: 000000000000002a 0000000000000000
ffff8801657abb48: ffff8801657abb88 xfs_trans_reserve_quota_bydquots+286
#10 [ffff8801657abb50] xfs_trans_reserve_quota_bydquots at ffffffffa03062ee
[xfs]
ffff8801657abb58: ffff8800c235df58 ffff8801a642dd80
ffff8801657abb68: ffff880413c93800 ffff8801657abc70
ffff8801657abb78: 000000000000000a 0000000000000000
ffff8801657abb88: ffff8801657abc48 xfs_create+546
#11 [ffff8801657abb90] xfs_create at ffffffffa02e76f2 [xfs]
ffff8801657abb98: 0000000000000001 ffff880100010000
ffff8801657abba8: ffff8801657abbd8 ffff8801657abc68
ffff8801657abbb8: ffff880413c93b48 0000000000000000
ffff8801657abbc8: 0000002a000081a4 __d_instantiate+146
ffff8801657abbd8: 0000000000000000 [cfq_queue]
ffff8801657abbe8: ffffffffffffffff [xfs_dquot]
ffff8801657abbf8: 0000000000000000 ffff8800c235df58
ffff8801657abc08: 0000000000000000 0000000000000000
ffff8801657abc18: 0000000051c3d868 ffff8801610e1680
ffff8801657abc28: 0000000000000000 ffff8801a642df38
ffff8801657abc38: 00000000000081a4 00000000000081b6
ffff8801657abc48: ffff8801657abcb0 xfs_vn_mknod+185
#12 [ffff8801657abc50] xfs_vn_mknod at ffffffffa02e3e99 [xfs]
ffff8801657abc58: generic_permission+272 ffff880100000000
ffff8801657abc68: 0000000000000000 ffff8801610e16b8
ffff8801657abc78: 0000000100000015 0000000051c3d868
ffff8801657abc88: 0000000000000000 ffff8801a642df38
ffff8801657abc98: ffff8801610e1680 00000000000081b6
ffff8801657abca8: 0000000000000000 ffff8801657abcc0
ffff8801657abcb8: xfs_vn_create+19
#13 [ffff8801657abcb8] xfs_vn_create at ffffffffa02e4043 [xfs]
ffff8801657abcc0: ffff8801657abcf8 vfs_create+140
#14 [ffff8801657abcc8] vfs_create at ffffffff81207b7c
ffff8801657abcd0: ffff8801657abf28 0000000000008241
ffff8801657abce0: ffff8801657abe50 ffff8801013255c0
ffff8801657abcf0: ffff8801610e1680 ffff8801657abda0
ffff8801657abd00: do_last+3085
#15 [ffff8801657abd00] do_last at ffffffff812096ed
ffff8801657abd08: ffffea000d264240 0000000000000000
ffff8801657abd18: ffff8801657abd68 ffff8801657abd90
ffff8801657abd28: ffff8801a642df38 [kmalloc-4096]
ffff8801657abd38: ffff88001f9cd000 ffff8801657abde4
ffff8801657abd48: ffff8801013255c0 0100000100000022
ffff8801657abd58: ffff8801657abdf0 00ff88001f9cd000
ffff8801657abd68: [kmalloc-192] 0000000051c3d868
ffff8801657abd78: ffff8801657abe50 ffff88001f9cd000
ffff8801657abd88: [kmalloc-4096] ffff8801657abf28
ffff8801657abd98: ffff88003914e580 ffff8801657abe40
ffff8801657abda8: path_openat+194
#16 [ffff8801657abda8] path_openat at ffffffff8120ab12
ffff8801657abdb0: ffff8801657abe80 user_path_at_empty+114
ffff8801657abdc0: ffff880413bcb720 ffff8801610e1680
ffff8801657abdd0: 0000001553594ce5 00000041f5e18020
ffff8801657abde0: 0000000100000000 0000000000000000
ffff8801657abdf0: ffff8801a642df38 0000000200000000
ffff8801657abe00: 0000000000000000 00007f1068e7c000
ffff8801657abe10: 0000000051c3d868 00000000ffffff9c
ffff8801657abe20: [kmalloc-4096] ffff8801657abf28
ffff8801657abe30: 0000000000000001 0000000000000023
ffff8801657abe40: ffff8801657abf10 do_filp_open+75
#17 [ffff8801657abe48] do_filp_open at ffffffff8120d82b
ffff8801657abe50: ffff880413bcb720 ffff8801013255c0
ffff8801657abe60: 0000001553594ce5 [kmalloc-4096]
ffff8801657abe70: 0000000000000000 00007ffd6cb58ad0
ffff8801657abe80: ffff8801a642df38 0000000200000301
ffff8801657abe90: 0000000000000000 0000000000000001
ffff8801657abea0: 00007ffd6cb58ad0 0000000000000000
ffff8801657abeb0: 0000000000000000 ffff8801657abf00
ffff8801657abec0: __alloc_fd+167 ffff880413ca91c0
ffff8801657abed0: 0000ffff00008241 0000000000000001
ffff8801657abee0: 0000000000008241 0000000051c3d868
ffff8801657abef0: 0000000000000001 0000000000000005
ffff8801657abf00: 00000000ffffff9c [kmalloc-4096]
ffff8801657abf10: ffff8801657abf68 do_sys_open+243
#18 [ffff8801657abf18] do_sys_open at ffffffff811fa3a3
ffff8801657abf20: ffff880286c5b648 ffff81b600008241
ffff8801657abf30: 0000030000000022 0000000051c3d868
ffff8801657abf40: 000000000041cf0f 0000000001aa8300
ffff8801657abf50: 0000000000000004 0000000000000001
ffff8801657abf60: 0000000000000023 ffff8801657abf78
ffff8801657abf70: sys_open+30
#19 [ffff8801657abf70] sys_open at ffffffff811fa4be
ffff8801657abf78: 00007ffd6cb58a60 system_call_fastpath+22
#20 [ffff8801657abf80] system_call_fastpath at ffffffff81645189
RIP: 00007f1067c3c850 RSP: 00007ffd6cb59bd8 RFLAGS: 00000246
RAX: 0000000000000002 RBX: ffffffff81645189 RCX: ffffffffffffffff
RDX: 00000000000001b6 RSI: 0000000000000241 RDI: 00007ffd6cb58ad0
RBP: 00007ffd6cb58a60 R8: 000000000041cf15 R9: 0000000000000240
R10: 0000000000000024 R11: 0000000000000246 R12: ffffffff811fa4be
R13: ffff8801657abf78 R14: 0000000000000023 R15: 0000000000000001
ORIG_RAX: 0000000000000002 CS: 0033 SS: 002b
crash>
I was trying to analyse the coredump and found a strange flag value at pdquot:
crash> struct xfs_mount -x ffff880413c93800
struct xfs_mount {
m_super = 0xffff880413c93000,
m_tid = 0x0,
m_ail = 0xffff880416fc7d80,
m_sb = {
sb_magicnum = 0x58465342,
sb_blocksize = 0x1000,
sb_dblocks = 0xda28800,
sb_rblocks = 0x0,
sb_rextents = 0x0,
sb_uuid = {
..
crash> struct xfs_mount.m_qflags -x ffff880413c93800
m_qflags = 0x560f
crash> struct xfs_inode -x ffff8801a642dd80
struct xfs_inode {
i_mount = 0xffff880413c93800,
i_udquot = 0xffff88035e460000,
i_gdquot = 0x0,
i_pdquot = 0xffff8800c235df58,
i_ino = 0x833d6095,
i_imap = {
im_blkno = 0x3828d040,
im_len = 0x10,
im_boffset = 0x1500
},
...
crash> struct xfs_dquot -x 0xffff88035e460000
struct xfs_dquot {
dq_flags = 0x1,
q_lru = {
next = 0xffff88035e460008,
prev = 0xffff88035e460008
},
q_mount = 0xffff880413c93800,
...
crash> struct xfs_dquot -x 0xffff8800c235df58
struct xfs_dquot {
dq_flags = 0xc235e308, <<--------------------------- ???????
q_lru = {
next = 0xffff8800c235df60,
prev = 0xffff8800c235df60
},
q_mount = 0xffff880413c93800,
...
The value dq_flags=0xc235e308 looks like a part of address.
Would you please help to find the root cause of the issue?
Thank you.