Bugzilla – Bug 646
[2.4.31][XFS] I cannot create XFS on evms volume larger than 1TB
Last modified: 2006-04-26 23:42:08 CDT
Hi Recently I ran into a problem that I cannot create XFS file system on the evms volume larger than 1TB. This is the message: # mkfs.xfs -f /dev/evms/volume1 meta-data=/dev/evms/volume1 isize=256 agcount=4294967134, agsize=1048576 blks = sectsz=512 data = bsize=4096 blocks=4503599456624639, imaxpct=25 = sunit=0 swidth=0 blks, unwritten=1 naming =version 2 bsize=4096 log =internal log bsize=4096 blocks=32768, version=1 = sectsz=512 sunit=0 blks realtime =none extsz=65536 blocks=0, rtextents=0 mkfs.xfs: pwrite64 failed: Invalid argument After tracing xfsprogs-2.5.6/xfs_mkfs.c, I found that program exits in line 1951: 1944 /* 1945 * Zero out the last 64k on the device, to obliterate any 1946 * old MD RAID (or other) metadata at the end of the device. 1947 */ 1948 if (!xi.disfile) { 1949 buf = libxfs_getbuf(xi.ddev, (xi.dsize - BTOBB(65536)), 1950 BTOBB(65536)); 1951 bzero(XFS_BUF_PTR(buf), 65536); I also found that XFS_BUF_PTR is defined in xfs/linux-2.4/xfs_buf.h And these are the strace of "mkfs.xfs /dev/evms/volume1" Thanks, execve("/usr/sbin/mkfs.xfs", ["mkfs.xfs", "-f", "/dev/evms/volume1"], [/* 14 vars */]) = 0 brk(0) = 0x809278c old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x40015000 open("/etc/ld.so.preload", O_RDONLY) = -1 ENOENT (No such file or directory) open("/etc/ld.so.cache", O_RDONLY) = -1 ENOENT (No such file or directory) open("/lib/i686/mmx/libc.so.6", O_RDONLY) = -1 ENOENT (No such file or directory) stat64("/lib/i686/mmx", 0xbffff0ec) = -1 ENOENT (No such file or directory) open("/lib/i686/libc.so.6", O_RDONLY) = -1 ENOENT (No such file or directory) stat64("/lib/i686", 0xbffff0ec) = -1 ENOENT (No such file or directory) open("/lib/mmx/libc.so.6", O_RDONLY) = -1 ENOENT (No such file or directory) stat64("/lib/mmx", 0xbffff0ec) = -1 ENOENT (No such file or directory) open("/lib/libc.so.6", O_RDONLY) = 3 read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\207\1"..., 1024) = 1024 fstat64(3, {st_mode=S_IFREG|0755, st_size=1247852, ...}) = 0 old_mmap(NULL, 1223712, PROT_READ|PROT_EXEC, MAP_PRIVATE, 3, 0) = 0x40016000 mprotect(0x40137000, 39968, PROT_NONE) = 0 old_mmap(0x40137000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 3, 0x120000) = 0x40137000 old_mmap(0x4013d000, 15392, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x4013d000 close(3) = 0 brk(0) = 0x809278c brk(0x80927b4) = 0x80927b4 brk(0x8093000) = 0x8093000 stat64("/dev/evms/volume1", {st_mode=S_IFBLK|0640, st_rdev=makedev(254, 21), ...}) = 0 open("/proc/devices", O_RDONLY|O_LARGEFILE) = 3 fstat64(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0 old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x40141000 read(3, "Character devices:\n 1 mem\n 2 p"..., 4096) = 246 read(3, "", 4096) = 0 close(3) = 0 munmap(0x40141000, 4096) = 0 open("/proc/devices", O_RDONLY|O_LARGEFILE) = 3 fstat64(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0 old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x40141000 read(3, "Character devices:\n 1 mem\n 2 p"..., 4096) = 246 read(3, "", 4096) = 0 close(3) = 0 munmap(0x40141000, 4096) = 0 open("/proc/devices", O_RDONLY|O_LARGEFILE) = 3 fstat64(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0 old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x40141000 read(3, "Character devices:\n 1 mem\n 2 p"..., 4096) = 246 read(3, "", 4096) = 0 close(3) = 0 munmap(0x40141000, 4096) = 0 open("/proc/devices", O_RDONLY|O_LARGEFILE) = 3 fstat64(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0 old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x40141000 read(3, "Character devices:\n 1 mem\n 2 p"..., 4096) = 246 read(3, "", 4096) = 0 close(3) = 0 munmap(0x40141000, 4096) = 0 getcwd("/root", 4096) = 6 stat64("/dev/evms/volume1", {st_mode=S_IFBLK|0640, st_rdev=makedev(254, 21), ...}) = 0 stat64("/dev/evms/volume1", {st_mode=S_IFBLK|0640, st_rdev=makedev(254, 21), ...}) = 0 ustat(0xfe15, 0xbfffe6e0) = -1 EINVAL (Invalid argument) open("/dev/evms/volume1", O_RDONLY|O_LARGEFILE) = 3 stat64("/dev/evms/volume1", {st_mode=S_IFBLK|0640, st_rdev=makedev(254, 21), ...}) = 0 stat64("/dev/evms/volume1", {st_mode=S_IFBLK|0640, st_rdev=makedev(254, 21), ...}) = 0 ustat(0xfe15, 0xbfffe6e0) = -1 EINVAL (Invalid argument) open("/dev/evms/volume1", O_RDWR|O_LARGEFILE) = 4 stat64("/dev/evms/volume1", {st_mode=S_IFBLK|0640, st_rdev=makedev(254, 21), ...}) = 0 ioctl(4, 0x40041271, 0xbfffe708) = 0 fstat64(4, {st_mode=S_IFBLK|0640, st_rdev=makedev(254, 21), ...}) = 0 ioctl(4, 0x80041272, 0xbfffe718) = 0 ioctl(4, BLKSSZGET, 0xbffffb00) = 0 chdir("/root") = 0 close(3) = 0 stat64("/dev/evms/volume1", {st_mode=S_IFBLK|0640, st_rdev=makedev(254, 21), ...}) = 0 open("/proc/devices", O_RDONLY|O_LARGEFILE) = 3 fstat64(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0 old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x40141000 read(3, "Character devices:\n 1 mem\n 2 p"..., 4096) = 246 read(3, "", 4096) = 0 close(3) = 0 munmap(0x40141000, 4096) = 0 open("/proc/devices", O_RDONLY|O_LARGEFILE) = 3 fstat64(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0 old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x40141000 read(3, "Character devices:\n 1 mem\n 2 p"..., 4096) = 246 read(3, "", 4096) = 0 close(3) = 0 munmap(0x40141000, 4096) = 0 open("/proc/devices", O_RDONLY|O_LARGEFILE) = 3 fstat64(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0 old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x40141000 read(3, "Character devices:\n 1 mem\n 2 p"..., 4096) = 246 read(3, "", 4096) = 0 close(3) = 0 munmap(0x40141000, 4096) = 0 open("/proc/devices", O_RDONLY|O_LARGEFILE) = 3 fstat64(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0 old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x40141000 read(3, "Character devices:\n 1 mem\n 2 p"..., 4096) = 246 read(3, "", 4096) = 0 close(3) = 0 munmap(0x40141000, 4096) = 0 fstat64(1, {st_mode=S_IFREG|0644, st_size=5875, ...}) = 0 old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x40141000 gettimeofday({1146045621, 337377}, NULL) = 0 open("/dev/urandom", O_RDONLY) = 3 getpid() = 8052 getuid32() = 0 gettimeofday({1146045621, 337578}, NULL) = 0 gettimeofday({1146045621, 337626}, NULL) = 0 read(3, "\355(\33JG\324\2128<\314\276\355\27\237\203r", 16) = 16 brk(0x80a5000) = 0x80a5000 pwrite(4, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 69632, 0) = 69632 pwrite(4, "XFSB\0\0\20\0\0\17\377\377\365\322\237\377\0\0\0\0\0\0"..., 512, 0) = 512 pwrite(4, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 65536, 18446743374334458880) = -1 EINVAL (Invalid argument) write(2, "mkfs.xfs: pwrite64 failed: Inval"..., 44mkfs.xfs: pwrite64 failed: Invalid argument ) = 44 write(1, "meta-data=/dev/evms/volume1 "..., 530meta- data=/dev/evms/volume1 isize=256 agcount=4294967134, agsize=1048576 blks = sectsz=512 data = bsize=4096 blocks=4503599456624639, imaxpct=25 = sunit=0 swidth=0 blks, unwritten=1 naming =version 2 bsize=4096 log =internal log bsize=4096 blocks=32768, version=1 = sectsz=512 sunit=0 blks realtime =none extsz=65536 blocks=0, rtextents=0 ) = 530 munmap(0x40141000, 4096) = 0 _exit(1) = ? I download xfsprogs-2.7.11.src.tar.gz and it still failed: #/mkfs.xfs -f /dev/evms/volume1 meta-data=/dev/evms/volume1 isize=256 agcount=4294967134, agsize=1048576 blks data = bsize=4096 blocks=4503599456624639, imaxpct=25 = sunit=0 swidth=0 blks, unwritten=0 naming =version 2 bsize=4096 log =internal log bsize=4096 blocks=32768, version=1 = sunit=0 blks realtime =none extsz=65536 blocks=0, rtextents=0 mkfs.xfs: lseek64 to 18446743374334458880 failed: Invalid argument Can anyone tell me that which part should I trace/debug ? What is wrong with "bzero(XFS_BUF_PTR(buf), 65536)"? Thanks -- Best Regards, Michael Yao
I am sorry, I want to correct that it exits "after" line 1951: 1952 "libxfs_writebuf(buf, LIBXFS_EXIT_ON_FAILURE);" What is wrong with "libxfs_writebuf(buf, LIBXFS_EXIT_ON_FAILURE)" ? Thanks
Is this reproducible on 2.6? Is CONFIG_LBD enabled? cheers.
Sorry I forgot to mention that in the example, the volume size is only 1.36 TB. I tried to make XFS on 1024GB volume, and it worked. I tried to make XFS on 1025GB volume, and it failed with same error message. I can't find CONFIG_LBD in my .config, can you tell me where/how to set CONFIG_LBD? I don't know it could be reproducible or not in kernel 2.6, I can try it later.
the LBD patch is not in mainstream 2.4 kernels, so you'll have an absolute 2T limit on any block device, 1T limits on some devices is not surprising. See if you can use any method to write to the last block on your device; I'm guessing it'll fail. XFS may just expose this underlying problem because it's trying to zero out those last few blocks.
I do not think this is a underlying problem because I can create EXT3, but I cannot create XFS on 1.36TB volume. I found that "buf" in xfsprogs-2.5.6/xfs_mkfs.c line#1949 is wrong, because I found that buf->b_blkno=18446743374334458880 (IO_DEBUG): #/mkfs.xfs -f /dev/evms/volume1 meta-data=/dev/evms/volume1 isize=256 agcount=4294967134, agsize=1048576 blks = sectsz=512 data = bsize=4096 blocks=4503599456624639, imaxpct=25 = sunit=0 swidth=0 blks, unwritten=1 naming =version 2 bsize=4096 log =internal log bsize=4096 blocks=32768, version=1 = sectsz=512 sunit=0 blks realtime =none extsz=65536 blocks=0, rtextents=0 getbuf allocated 69632bytes, blkno=0(0), 0x8092918 writing 69632bytes at blkno=0(0), 0x8092918 putbuf released 69632bytes, 0x8092918 getbuf allocated 512bytes, blkno=0(0), 0x8092918 writing 512bytes at blkno=0(0), 0x8092918 putbuf released 512bytes, 0x8092918 getbuf allocated 65536bytes, blkno=18446743374334458880(36028795652996990), 0x8092918 writing 65536bytes at blkno=18446743374334458880(36028795652996990), 0x8092918 mkfs.xfs: pwrite64 failed: Invalid argument So I think there maybe some problem in function libxfs_getbuf in libxfs/rdwr.c. I can create XFS successfully when I skipped line 1948~1952 and assigned size: #mkfs.xfs -f -L volume1 -d size=1392g /dev/evms/volume1 meta-data=/dev/evms/volume1 isize=256 agcount=348, agsize=1048576 blks = sectsz=512 data = bsize=4096 blocks=364904448, imaxpct=25 = sunit=0 swidth=0 blks, unwritten=1 naming =version 2 bsize=4096 log =internal log bsize=4096 blocks=32768, version=1 = sectsz=512 sunit=0 blks realtime =none extsz=65536 blocks=0, rtextents=0
Can you please try a newer xfsprogs to see if the problem persists? 2.5.6 is almost 3 years old now. xfsprogs in cvs is now at 2.7.16. Thanks, -Eric
Oh, I'm sorry, I see that you tried 2.7.11. Guess I should read the whole bug :)
mkfs.xfs uses the BLKGETSIZE64 ioctl to get the device size; can you instrument the platform_findsizes() function to see what value mkfs is getting from the ioctl (or write your own ioctl test)? error = ioctl(fd, BLKGETSIZE64, &size); returns "size" in bytes.
this looks like a devicemapper bug, in dm_blk_ioctl() they're assigning a full unsigned 32 bits of device size to a signed 32 bit size, and start running into trouble with sign extension when they cast it up. Changing "long size;" to a sector_t (defined as unsigned long in the dm patch) might fix this up, or whatever it takes to get the sizes & signs right in that function. If you can convince yourself that this really still is an xfs bug feel free to reopen :)