How to add the lustre file system client to a BlueData container? - lustre
I'm trying to set up a lustre client (docs) inside a docker container running on BlueData.
As per this post, I've modified the BlueData config on each worker and the controller node:
$ vi /opt/bluedata/common-install/bd_mgmt/releases/1/sys.config
I added the SYS_ADMIN capability:
{allowed_docker_caps, ["SETPCAP",
"SYS_ADMIN",
...
And rebooted the host.
Next, I provisioned a Centos 7.x cluster in BlueData:
CentOS 7.x with no pre-packaged apps or software
Image Version: 2.2
Distro ID: bluedata/centos7
Then I ssh'd into the Centos container:
$ ssh -o StrictHostKeyChecking=no -i /Users/me/.ssh/id_rsa centos#x.x.x.x
Inside the container, I install the lustre client:
sudo yum install \
kernel \
kernel-devel \
kernel-headers \
kernel-abi-whitelists \
kernel-tools \
kernel-tools-libs \
kernel-tools-libs-devel
cat >/tmp/lustre-repo.conf <<\__EOF
[lustre-server]
name=lustre-server
baseurl=https://downloads.whamcloud.com/public/lustre/latest-release/el7/server
gpgcheck=0
[lustre-client]
name=lustre-client
baseurl=https://downloads.whamcloud.com/public/lustre/latest-release/el7/client
gpgcheck=0
[e2fsprogs-wc]
name=e2fsprogs-wc
baseurl=https://downloads.whamcloud.com/public/e2fsprogs/latest/el7
gpgcheck=0
__EOF
sudo mv /tmp/lustre-repo.conf /etc/yum.repos.d/lustre.repo
sudo reboot
sudo yum install epel-release
sudo yum --nogpgcheck --enablerepo=lustre-client install lustre-client-dkms lustre-client
sudo reboot
However, I receive an error when I try to load the lustre module:
$ sudo modprobe -v lustre
insmod /lib/modules/3.10.0-957.21.3.el7.x86_64/kernel/crypto/crct10dif_generic.ko.xz
insmod /lib/modules/3.10.0-957.21.3.el7.x86_64/kernel/lib/crc-t10dif.ko.xz
modprobe: ERROR: could not insert 'lustre': Operation not permitted
I have checked the kernel version:
[bluedata#bluedata-2 ~]$ uname -a
Linux bluedata-2.bdlocal 3.10.0-957.21.3.el7.x86_64 #1 SMP Tue Jun 18 16:35:19 UTC 2019 x86_64 x86_64 x86_64 GNU/Linux
The lustre version I installed is 2.12:
kmod-lustre-client.x86_64 2.12.2-1.el7 #lustre-client
lustre-client.x86_64 2.12.2-1.el7 #lustre-client
Update 1
No errors are shown with dmesg:
[bluedata#bluedata-3 ~]$ dmesg -c
[bluedata#bluedata-3 ~]$ sudo modprobe -v lustre
insmod /lib/modules/3.10.0-957.21.3.el7.x86_64/kernel/crypto/crct10dif_generic.ko.xz
insmod /lib/modules/3.10.0-957.21.3.el7.x86_64/kernel/lib/crc-t10dif.ko.xz
modprobe: ERROR: could not insert 'lustre': Operation not permitted
[bluedata#bluedata-3 ~]$ dmesg
Update 2
$ sudo strace modprobe lustre
Outputs:
execve("/sbin/modprobe", ["modprobe", "lustre"], [/* 16 vars */]) = 0
brk(NULL) = 0x1648000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f4458ff2000
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=22387, ...}) = 0
mmap(NULL, 22387, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f4458fec000
close(3) = 0
open("/lib64/liblzma.so.5", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\2000\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=157424, ...}) = 0
mmap(NULL, 2249352, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f4458bac000
mprotect(0x7f4458bd1000, 2093056, PROT_NONE) = 0
mmap(0x7f4458dd0000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x24000) = 0x7f4458dd0000
close(3) = 0
open("/lib64/libz.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\20!\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=90248, ...}) = 0
mmap(NULL, 2183272, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f4458996000
mprotect(0x7f44589ab000, 2093056, PROT_NONE) = 0
mmap(0x7f4458baa000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14000) = 0x7f4458baa000
close(3) = 0
open("/lib64/libgcc_s.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\220*\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=88776, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f4458feb000
mmap(NULL, 2184192, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f4458780000
mprotect(0x7f4458795000, 2093056, PROT_NONE) = 0
mmap(0x7f4458994000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14000) = 0x7f4458994000
close(3) = 0
open("/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240%\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=2151672, ...}) = 0
mmap(NULL, 3981792, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f44583b3000
mprotect(0x7f4458575000, 2097152, PROT_NONE) = 0
mmap(0x7f4458775000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1c2000) = 0x7f4458775000
mmap(0x7f445877b000, 16864, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f445877b000
close(3) = 0
open("/lib64/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260l\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=141968, ...}) = 0
mmap(NULL, 2208904, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f4458197000
mprotect(0x7f44581ae000, 2093056, PROT_NONE) = 0
mmap(0x7f44583ad000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x16000) = 0x7f44583ad000
mmap(0x7f44583af000, 13448, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f44583af000
close(3) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f4458fea000
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f4458fe8000
arch_prctl(ARCH_SET_FS, 0x7f4458fe8740) = 0
mprotect(0x7f4458775000, 16384, PROT_READ) = 0
mprotect(0x7f44583ad000, 4096, PROT_READ) = 0
mprotect(0x7f4458994000, 4096, PROT_READ) = 0
mprotect(0x7f4458baa000, 4096, PROT_READ) = 0
mprotect(0x7f4458dd0000, 4096, PROT_READ) = 0
mprotect(0x621000, 4096, PROT_READ) = 0
mprotect(0x7f4458ff3000, 4096, PROT_READ) = 0
munmap(0x7f4458fec000, 22387) = 0
set_tid_address(0x7f4458fe8a10) = 1264
set_robust_list(0x7f4458fe8a20, 24) = 0
rt_sigaction(SIGRTMIN, {0x7f445819d790, [], SA_RESTORER|SA_SIGINFO, 0x7f44581a65d0}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {0x7f445819d820, [], SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x7f44581a65d0}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
brk(NULL) = 0x1648000
brk(0x1669000) = 0x1669000
brk(NULL) = 0x1669000
uname({sysname="Linux", nodename="bluedata-3.bdlocal", ...}) = 0
stat("/etc/modprobe.d", {st_mode=S_IFDIR|0755, st_size=54, ...}) = 0
openat(AT_FDCWD, "/etc/modprobe.d", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 3
getdents(3, /* 4 entries */, 32768) = 128
newfstatat(3, "dccp-blacklist.conf", {st_mode=S_IFREG|0644, st_size=215, ...}, 0) = 0
newfstatat(3, "ko2iblnd.conf", {st_mode=S_IFREG|0644, st_size=999, ...}, 0) = 0
getdents(3, /* 0 entries */, 32768) = 0
close(3) = 0
stat("/run/modprobe.d", 0x7ffcc1e0a640) = -1 ENOENT (No such file or directory)
stat("/lib/modprobe.d", {st_mode=S_IFDIR|0755, st_size=6, ...}) = 0
openat(AT_FDCWD, "/lib/modprobe.d", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 3
getdents(3, /* 2 entries */, 32768) = 48
getdents(3, /* 0 entries */, 32768) = 0
close(3) = 0
open("/etc/modprobe.d/dccp-blacklist.conf", O_RDONLY|O_CLOEXEC) = 3
fcntl(3, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(3, {st_mode=S_IFREG|0644, st_size=215, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f4458ff1000
read(3, "# DCCP is considered a potential"..., 4096) = 215
read(3, "", 4096) = 0
close(3) = 0
munmap(0x7f4458ff1000, 4096) = 0
open("/etc/modprobe.d/ko2iblnd.conf", O_RDONLY|O_CLOEXEC) = 3
fcntl(3, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(3, {st_mode=S_IFREG|0644, st_size=999, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f4458ff1000
read(3, "# Currently it isn't possible to"..., 4096) = 999
read(3, "", 4096) = 0
close(3) = 0
munmap(0x7f4458ff1000, 4096) = 0
open("/lib/modules/3.10.0-957.21.3.el7.x86_64/modules.softdep", O_RDONLY|O_CLOEXEC) = 3
fcntl(3, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(3, {st_mode=S_IFREG|0644, st_size=518, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f4458ff1000
read(3, "# Soft dependencies extracted fr"..., 4096) = 518
read(3, "", 4096) = 0
close(3) = 0
munmap(0x7f4458ff1000, 4096) = 0
open("/proc/cmdline", O_RDONLY|O_CLOEXEC) = 3
read(3, "BOOT_IMAGE=/boot/vmlinuz-3.10.0-"..., 4095) = 193
read(3, "", 3902) = 0
close(3) = 0
open("/lib/modules/3.10.0-957.21.3.el7.x86_64/modules.dep.bin", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=382199, ...}) = 0
mmap(NULL, 382199, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f4458f8a000
close(3) = 0
open("/lib/modules/3.10.0-957.21.3.el7.x86_64/modules.alias.bin", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=802187, ...}) = 0
mmap(NULL, 802187, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f4458ec6000
close(3) = 0
open("/lib/modules/3.10.0-957.21.3.el7.x86_64/modules.symbols.bin", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=537967, ...}) = 0
mmap(NULL, 537967, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f4458e42000
close(3) = 0
open("/lib/modules/3.10.0-957.21.3.el7.x86_64/modules.builtin.bin", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=9332, ...}) = 0
mmap(NULL, 9332, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f4458fef000
close(3) = 0
stat("/lib/modules/3.10.0-957.21.3.el7.x86_64/extra/lmv.ko.xz", {st_mode=S_IFREG|0644, st_size=58688, ...}) = 0
stat("/lib/modules/3.10.0-957.21.3.el7.x86_64/extra/mdc.ko.xz", {st_mode=S_IFREG|0644, st_size=81772, ...}) = 0
stat("/lib/modules/3.10.0-957.21.3.el7.x86_64/extra/fid.ko.xz", {st_mode=S_IFREG|0644, st_size=11592, ...}) = 0
stat("/lib/modules/3.10.0-957.21.3.el7.x86_64/extra/osc.ko.xz", {st_mode=S_IFREG|0644, st_size=133688, ...}) = 0
stat("/lib/modules/3.10.0-957.21.3.el7.x86_64/extra/lov.ko.xz", {st_mode=S_IFREG|0644, st_size=101472, ...}) = 0
stat("/lib/modules/3.10.0-957.21.3.el7.x86_64/extra/fld.ko.xz", {st_mode=S_IFREG|0644, st_size=14600, ...}) = 0
stat("/lib/modules/3.10.0-957.21.3.el7.x86_64/extra/ptlrpc.ko.xz", {st_mode=S_IFREG|0644, st_size=369448, ...}) = 0
stat("/lib/modules/3.10.0-957.21.3.el7.x86_64/extra/obdclass.ko.xz", {st_mode=S_IFREG|0644, st_size=270652, ...}) = 0
stat("/lib/modules/3.10.0-957.21.3.el7.x86_64/extra/lnet.ko.xz", {st_mode=S_IFREG|0644, st_size=174800, ...}) = 0
stat("/lib/modules/3.10.0-957.21.3.el7.x86_64/extra/libcfs.ko.xz", {st_mode=S_IFREG|0644, st_size=88252, ...}) = 0
stat("/lib/modules/3.10.0-957.21.3.el7.x86_64/kernel/lib/crc-t10dif.ko.xz", {st_mode=S_IFREG|0644, st_size=2028, ...}) = 0
stat("/lib/modules/3.10.0-957.21.3.el7.x86_64/kernel/crypto/crct10dif_common.ko.xz", {st_mode=S_IFREG|0644, st_size=2004, ...}) = 0
open("/sys/module/lustre/initstate", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/sys/module/lustre", 0x7ffcc1e0a5c0) = -1 ENOENT (No such file or directory)
stat("/lib/modules/3.10.0-957.21.3.el7.x86_64/kernel/crypto/crct10dif_common.ko.xz", {st_mode=S_IFREG|0644, st_size=2004, ...}) = 0
stat("/lib/modules/3.10.0-957.21.3.el7.x86_64/kernel/crypto/crct10dif_common.ko.xz", {st_mode=S_IFREG|0644, st_size=2004, ...}) = 0
open("/sys/module/crct10dif_common/initstate", O_RDONLY|O_CLOEXEC) = 3
read(3, "live\n", 31) = 5
read(3, "", 26) = 0
close(3) = 0
open("/sys/module/crct10dif_common/initstate", O_RDONLY|O_CLOEXEC) = 3
read(3, "live\n", 31) = 5
read(3, "", 26) = 0
close(3) = 0
open("/sys/module/crct10dif_pclmul/initstate", O_RDONLY|O_CLOEXEC) = 3
read(3, "live\n", 31) = 5
read(3, "", 26) = 0
close(3) = 0
open("/sys/module/crct10dif_common/initstate", O_RDONLY|O_CLOEXEC) = 3
read(3, "live\n", 31) = 5
read(3, "", 26) = 0
close(3) = 0
open("/sys/module/crct10dif_generic/initstate", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/sys/module/crct10dif_generic", 0x7ffcc1e0a5c0) = -1 ENOENT (No such file or directory)
open("/lib/modules/3.10.0-957.21.3.el7.x86_64/kernel/crypto/crct10dif_generic.ko.xz", O_RDONLY|O_CLOEXEC) = 3
read(3, "\3757zXZ\0", 6) = 6
lseek(3, 0, SEEK_SET) = 0
read(3, "\3757zXZ\0\0\4\346\326\264F\2\0!\1\26\0\0\0t/\345\243\340\30l\6\267]\0?"..., 8192) = 1784
mmap(NULL, 8392704, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f4457996000
read(3, "", 8192) = 0
munmap(0x7f4457996000, 8392704) = 0
init_module(0x1653f40, 6253, "") = -1 ENOSYS (Function not implemented)
open("/sys/module/crc_t10dif/initstate", O_RDONLY|O_CLOEXEC) = -1 ENOSYS (Function not implemented)
stat("/sys/module/crc_t10dif", 0x7ffcc1e0a5c0) = -1 ENOSYS (Function not implemented)
open("/lib/modules/3.10.0-957.21.3.el7.x86_64/kernel/lib/crc-t10dif.ko.xz", O_RDONLY|O_CLOEXEC) = -1 ENOSYS (Function not implemented)
read(4, 0x7ffcc1e0b5f0, 6) = -1 ENOSYS (Function not implemented)
lseek(4, 0, SEEK_SET) = -1 ENOSYS (Function not implemented)
read(4, 0x7ffcc1e074e0, 8192) = -1 ENOSYS (Function not implemented)
brk(NULL) = -1 ENOSYS (Function not implemented)
brk(0x1e7d000) = -1 ENOSYS (Function not implemented)
read(4, 0x7ffcc1e074e0, 8192) = -1 EPERM (Operation not permitted)
close(3) = 0
write(2, "modprobe: ERROR: could not inser"..., 68modprobe: ERROR: could not insert 'lustre': Operation not permitted
) = 68
close(4) = 0
munmap(0x7f4458f8a000, 382199) = 0
munmap(0x7f4458ec6000, 802187) = 0
munmap(0x7f4458e42000, 537967) = 0
munmap(0x7f4458fef000, 9332) = 0
exit_group(1) = ?
+++ exited with 1 +++
Update 3
I tried installing the kmod package instead of dkms:
Running transaction
Installing : kmod-lustre-client-2.12.2-1.el7.x86_64 1/1
mknod: '/var/tmp/dracut.cG1SKj/initramfs/dev/null': Operation not permitted
mknod: '/var/tmp/dracut.cG1SKj/initramfs/dev/kmsg': Operation not permitted
mknod: '/var/tmp/dracut.cG1SKj/initramfs/dev/console': Operation not permitted
Verifying : kmod-lustre-client-2.12.2-1.el7.x86_64 1/1
Installed:
kmod-lustre-client.x86_64 0:2.12.2-1.el7
Complete!
I then tried again sudo strace modprobe lustre:
...
open("/lib/modules/3.10.0-957.21.3.el7.x86_64/kernel/lib/crc-t10dif.ko.xz", O_RDONLY|O_CLOEXEC) = -1 ENOSYS (Function not implemented)
read(4, 0x7fff450be5f0, 6) = -1 ENOSYS (Function not implemented)
lseek(4, 0, SEEK_SET) = -1 ENOSYS (Function not implemented)
read(4, 0x7fff450ba4e0, 8192) = -1 ENOSYS (Function not implemented)
brk(NULL) = -1 ENOSYS (Function not implemented)
brk(0x1410000) = -1 ENOSYS (Function not implemented)
read(4, 0x7fff450ba4e0, 8192) = -1 EPERM (Operation not permitted)
close(3) = 0
write(2, "modprobe: ERROR: could not inser"..., 68modprobe: ERROR: could not insert 'lustre': Operation not permitted
) = 68
close(4) = 0
munmap(0x7f04da388000, 383873) = 0
munmap(0x7f04da2c4000, 802187) = 0
munmap(0x7f04da240000, 537967) = 0
munmap(0x7f04da3ed000, 9332) = 0
exit_group(1) = ?
+++ exited with 1 +++
Update 4
Running the container as --privileged has resolved the original error, but I now hit a new error:
[bluedata#bluedata-5 ~]$ sudo dmesg -c
[bluedata#bluedata-5 ~]$ sudo modprobe -v lustre
insmod /lib/modules/3.10.0-957.21.3.el7.x86_64/extra/ptlrpc.ko.xz
modprobe: ERROR: could not insert 'lustre': Invalid argument
[bluedata#bluedata-5 ~]$ dmesg
[ 2072.258326] LNetError: 56638:0:(api-ni.c:2233:lnet_startup_lndnet()) Can't load LND tcp, module ksocklnd, rc=256
[ 2072.264113] LustreError: 56638:0:(events.c:625:ptlrpc_init_portals()) network initialisation failed
Update 5
The error message suggested I needed to configure the network, so I tried:
[bluedata#bluedata-5 ~]$ sudo modprobe lnet
[bluedata#bluedata-5 ~]$ sudo lnetctl lnet configure
lustre now loads without error:
[bluedata#bluedata-5 ~]$ sudo modprobe -v lustre
insmod /lib/modules/3.10.0-957.21.3.el7.x86_64/extra/ptlrpc.ko.xz
insmod /lib/modules/3.10.0-957.21.3.el7.x86_64/extra/fld.ko.xz
insmod /lib/modules/3.10.0-957.21.3.el7.x86_64/extra/lov.ko.xz
insmod /lib/modules/3.10.0-957.21.3.el7.x86_64/extra/osc.ko.xz
insmod /lib/modules/3.10.0-957.21.3.el7.x86_64/extra/fid.ko.xz
insmod /lib/modules/3.10.0-957.21.3.el7.x86_64/extra/mdc.ko.xz
insmod /lib/modules/3.10.0-957.21.3.el7.x86_64/extra/lmv.ko.xz
insmod /lib/modules/3.10.0-957.21.3.el7.x86_64/extra/lustre.ko.xz
I followed the original steps in the question and run the container as --privileged. Then loading and configuring lnet allowed loading the lustre module without error:
[bluedata#bluedata-5 ~]$ sudo modprobe lnet
[bluedata#bluedata-5 ~]$ sudo lnetctl lnet configure
[bluedata#bluedata-5 ~]$ sudo modprobe -v lustre
insmod /lib/modules/3.10.0-957.21.3.el7.x86_64/extra/ptlrpc.ko.xz
insmod /lib/modules/3.10.0-957.21.3.el7.x86_64/extra/fld.ko.xz
insmod /lib/modules/3.10.0-957.21.3.el7.x86_64/extra/lov.ko.xz
insmod /lib/modules/3.10.0-957.21.3.el7.x86_64/extra/osc.ko.xz
insmod /lib/modules/3.10.0-957.21.3.el7.x86_64/extra/fid.ko.xz
insmod /lib/modules/3.10.0-957.21.3.el7.x86_64/extra/mdc.ko.xz
insmod /lib/modules/3.10.0-957.21.3.el7.x86_64/extra/lmv.ko.xz
insmod /lib/modules/3.10.0-957.21.3.el7.x86_64/extra/lustre.ko.xz
[bluedata#bluedata-5 ~]$
IMPORTANT NOTE: Running with the privileged flag is not recommended. There are other options - reach out to your local BlueData team to learn more.
Related
Perl using Wx yields core dump in debug mode (option -d)
I have the following 3 line example to show the error: use strict; use warnings; use Wx; It works fine when I run it with perl -w hue5.pl. However, when I add the debugging option and start if with perl -w -d hue5.pl, a problem emerges and perl runtime crashes saying: Loading DB routines from perl5db.pl version 1.55 Editor support available. Enter h or 'h h' for help, or 'man perldebug' for more help. Wx::CODE(0x5604c197ca80)(/usr/lib64/perl5/vendor_perl/Wx.pm:154): 154: UnsetConstants() if defined &UnsetConstants; double free or corruption (!prev) Aborted (core dumped) I use perl v5.30.3 on 64 bit Linux (up-to-date Fedora release 31) and Wx installed from the binary package perl-Wx-0.9932-15.fc31.x86_64.rpm. Is anybody else having this issue?
Short version: Uninstalling the package perl-Term-ReadLine-Gnu solved the issue.
Longer version: I saw that the issue does not always happen. In fact, it seems to depend on the size of the process environment. At one point of debugging I could make the crash appear by declaring in the bash (note: without export the issue does not show)
export SOMEVAR=xyz
and disappear by
unset SOMEVAR
Finally, during further experimentation, I once got a more elaborate failure message pointing right to Readline/Gnu:
Signal SEGV at /usr/lib64/perl5/vendor_perl/Term/ReadLine/Gnu.pm line 504.
Term::ReadLine::Gnu::ornaments(Term::ReadLine=HASH(0x5593c12e5190), 1) > called at /usr/lib64/perl5/vendor_perl/Term/ReadLine/Gnu.pm line 301
Term::ReadLine::Gnu::new("Term::ReadLine", "perldb", GLOB(0x5593c062df58), GLOB(0x5593c05c6ea8)) called at /usr/share/perl5/perl5db.pl line 6865
DB::setterm() called at /usr/share/perl5/perl5db.pl line 1849
DB::_DB__read_next_cmd(undef) called at /usr/share/perl5/perl5db.pl line 2789
DB::DB called at f.pl line 5
For the curious, I note that line 504 in /usr/lib64/perl5/vendor_perl/Term/ReadLine/Gnu.pm was:
return Term::ReadLine::Gnu::XS::ornaments(#_);
For the even more curious I add strace dumps generated by runs which differ only with regard to the absence/presence of SOMEVAR in the environment (note that the name does not seem to have importance, I first got into the direction of environment differences by observing different behaviour depending on OLDPWD).
First without the issue (i.e. without variable SOMEVAR), quoting the lines as of loading Readline/Gnu library:
stat("/usr/lib65/perl5/vendor_perl/auto/Term/ReadLine/Gnu/XS/autosplit.ix", {st_mode=S_IFREG|0644, st_size=1137, ...}) = 0
openat(AT_FDCWD, "/usr/lib64/perl5/vendor_perl/auto/Term/ReadLine/Gnu/XS/autosplit.ix", O_RDONLY|O_CLOEXEC) = 13
ioctl(13, TCGETS, 0x7fff0a1cac70) = -1 ENOTTY (Inappropriate ioctl for device)
lseek(13, 0, SEEK_CUR) = 0
read(13, "# Index created by AutoSplit for"..., 8192) = 1137
read(13, "", 8192) = 0
close(13) = 0
lseek(3, 2189, SEEK_SET) = 2189
lseek(3, 0, SEEK_CUR) = 2189
close(3) = 0
getuid() = 1000
geteuid() = 1000
getgid() = 1000
getegid() = 1000
fcntl(4, F_GETFL) = 0x8002 (flags O_RDWR|O_LARGEFILE)
fcntl(5, F_GETFL) = 0x8002 (flags O_RDWR|O_LARGEFILE)
ioctl(1, TCGETS, {B38400 opost isig icanon echo ...}) = 0
stat("/home/bernhard/.terminfo", 0x55ec16193000) = -1 ENOENT (No such file or directory)
stat("/etc/terminfo", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
stat("/usr/share/terminfo", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
access("/etc/terminfo/x/xterm-256color", R_OK) = -1 ENOENT (No such file or directory)
access("/usr/share/terminfo/x/xterm-256color", R_OK) = 0
openat(AT_FDCWD, "/usr/share/terminfo/x/xterm-256color", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=3808, ...}) = 0
read(3, "\36\2%\0&\0\17\0\235\0010\6xterm-256color|xterm"..., 32768) = 3808
read(3, "", 28672) = 0
close(3) = 0
ioctl(1, TCGETS, {B38400 opost isig icanon echo ...}) = 0
ioctl(1, TCGETS, {B38400 opost isig icanon echo ...}) = 0
ioctl(1, TCGETS, {B38400 opost isig icanon echo ...}) = 0
ioctl(1, TCGETS, {B38400 opost isig icanon echo ...}) = 0
ioctl(1, TIOCGWINSZ, {ws_row=43, ws_col=173, ws_xpixel=0, ws_ypixel=0}) = 0
ioctl(4, TIOCGWINSZ, {ws_row=43, ws_col=173, ws_xpixel=0, ws_ypixel=0}) = 0
ioctl(4, TIOCGWINSZ, {ws_row=43, ws_col=173, ws_xpixel=0, ws_ypixel=0}) = 0
ioctl(4, TIOCSWINSZ, {ws_row=43, ws_col=173, ws_xpixel=0, ws_ypixel=0}) = 0
ioctl(4, TCGETS, {B38400 opost isig icanon echo ...}) = 0
openat(AT_FDCWD, "/home/bernhard/.inputrc", O_RDONLY) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/etc/inputrc", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=943, ...}) = 0
read(3, "# do not bell on tab-completion\n"..., 943) = 943
close(3) = 0
brk(NULL) = 0x55ec1640f000
brk(0x55ec16430000) = 0x55ec16430000
stat("/usr/lib64/perl5/vendor_perl/auto/Term/ReadLine/Gnu/XS/ornaments.al", {st_mode=S_IFREG|0644, st_size=1351, ...}) = 0
geteuid() = 1000
geteuid() = 1000
getegid() = 1000
getgroups(0, NULL) = 2
getgroups(2, [18, 1000]) = 2
stat("/usr/lib64/perl5/vendor_perl/auto/Term/ReadLine/Gnu/XS/ornaments.al", {st_mode=S_IFREG|0644, st_size=1351, ...}) = 0
openat(AT_FDCWD, "/usr/lib64/perl5/vendor_perl/auto/Term/ReadLine/Gnu/XS/ornaments.al", O_RDONLY|O_CLOEXEC) = 3
ioctl(3, TCGETS, 0x7fff0a1cb9d0) = -1 ENOTTY (Inappropriate ioctl for device)
lseek(3, 0, SEEK_CUR) = 0
read(3, "# NOTE: Derived from blib/lib/Te"..., 8192) = 1351
read(3, "", 8192) = 0
close(3) = 0
getpid() = 4060
getpid() = 4060
ioctl(4, TIOCGWINSZ, {ws_row=43, ws_col=173, ws_xpixel=0, ws_ypixel=0}) = 0
ioctl(4, TIOCSWINSZ, {ws_row=43, ws_col=173, ws_xpixel=0, ws_ypixel=0}) = 0
ioctl(4, TCGETS, {B38400 opost isig icanon echo ...}) = 0
ioctl(4, SNDCTL_TMR_STOP or TCSETSW, {B38400 opost isig -icanon -echo ...}) = 0
rt_sigprocmask(SIG_BLOCK, [HUP INT QUIT ALRM TERM TSTP TTIN TTOU], [], 8) = 0
rt_sigaction(SIGINT, {sa_handler=0x7f4433da2e70, sa_mask=[], sa_flags=SA_RESTORER, sa_restorer=0x7f44490e46b0}, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=SA_RESTORER, sa_restorer=0x7f44490e46b0}, 8) = 0
rt_sigaction(SIGTERM, {sa_handler=0x7f4433da2e70, sa_mask=[], sa_flags=SA_RESTORER, sa_restorer=0x7f44490e46b0}, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0
rt_sigaction(SIGHUP, {sa_handler=0x7f4433da2e70, sa_mask=[], sa_flags=SA_RESTORER, sa_restorer=0x7f44490e46b0}, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0
rt_sigaction(SIGQUIT, {sa_handler=0x7f4433da2e70, sa_mask=[], sa_flags=SA_RESTORER, sa_restorer=0x7f44490e46b0}, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0
rt_sigaction(SIGALRM, {sa_handler=0x7f4433da2e70, sa_mask=[], sa_flags=SA_RESTORER, sa_restorer=0x7f44490e46b0}, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0
rt_sigaction(SIGTSTP, {sa_handler=0x7f4433da2e70, sa_mask=[], sa_flags=SA_RESTORER, sa_restorer=0x7f44490e46b0}, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0
rt_sigaction(SIGTTOU, {sa_handler=0x7f4433da2e70, sa_mask=[], sa_flags=SA_RESTORER, sa_restorer=0x7f44490e46b0}, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0
rt_sigaction(SIGTTIN, {sa_handler=0x7f4433da2e70, sa_mask=[], sa_flags=SA_RESTORER, sa_restorer=0x7f44490e46b0}, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
rt_sigaction(SIGWINCH, {sa_handler=0x7f4433da22f0, sa_mask=[], sa_flags=SA_RESTORER|SA_RESTART, sa_restorer=0x7f44490e46b0}, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0
fstat(5, {st_mode=S_IFCHR|0666, st_rdev=makedev(0x5, 0), ...}) = 0
ioctl(5, TCGETS, {B38400 opost isig -icanon -echo ...}) = 0
write(5, "\33[4m DB<1> \33[24m", 17) = 17
pselect6(5, [4], NULL, NULL, NULL, {[], 8}) = 1 (in [4])
read(4, "q", 1) = 1
select(5, [4], NULL, [4], {tv_sec=0, tv_usec=0}) = 0 (Timeout)
write(5, "q", 1) = 1
pselect6(5, [4], NULL, NULL, NULL, {[], 8}) = 1 (in [4])
read(4, "\r", 1) = 1
write(5, "\n", 1) = 1
ioctl(4, SNDCTL_TMR_STOP or TCSETSW, {B38400 opost isig icanon echo ...}) = 0
rt_sigaction(SIGINT, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=SA_RESTORER, sa_restorer=0x7f44490e46b0}, {sa_handler=0x7f4433da2e70, sa_mask=[], sa_flags=SA_RESTORER, sa_restorer=0x7f44490e46b0}, 8) = 0
rt_sigaction(SIGTERM, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=SA_RESTORER, sa_restorer=0x7f44490e46b0}, {sa_handler=0x7f4433da2e70, sa_mask=[], sa_flags=SA_RESTORER, sa_restorer=0x7f44490e46b0}, 8) = 0
rt_sigaction(SIGHUP, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=SA_RESTORER, sa_restorer=0x7f44490e46b0}, {sa_handler=0x7f4433da2e70, sa_mask=[], sa_flags=SA_RESTORER, sa_restorer=0x7f44490e46b0}, 8) = 0
rt_sigaction(SIGQUIT, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=SA_RESTORER, sa_restorer=0x7f44490e46b0}, {sa_handler=0x7f4433da2e70, sa_mask=[], sa_flags=SA_RESTORER, sa_restorer=0x7f44490e46b0}, 8) = 0
rt_sigaction(SIGALRM, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=SA_RESTORER, sa_restorer=0x7f44490e46b0}, {sa_handler=0x7f4433da2e70, sa_mask=[], sa_flags=SA_RESTORER, sa_restorer=0x7f44490e46b0}, 8) = 0
rt_sigaction(SIGTSTP, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=SA_RESTORER, sa_restorer=0x7f44490e46b0}, {sa_handler=0x7f4433da2e70, sa_mask=[], sa_flags=SA_RESTORER, sa_restorer=0x7f44490e46b0}, 8) = 0
rt_sigaction(SIGTTOU, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=SA_RESTORER, sa_restorer=0x7f44490e46b0}, {sa_handler=0x7f4433da2e70, sa_mask=[], sa_flags=SA_RESTORER, sa_restorer=0x7f44490e46b0}, 8) = 0
rt_sigaction(SIGTTIN, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=SA_RESTORER, sa_restorer=0x7f44490e46b0}, {sa_handler=0x7f4433da2e70, sa_mask=[], sa_flags=SA_RESTORER, sa_restorer=0x7f44490e46b0}, 8) = 0
rt_sigaction(SIGWINCH, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=SA_RESTORER, sa_restorer=0x7f44490e46b0}, {sa_handler=0x7f4433da22f0, sa_mask=[], sa_flags=SA_RESTORER|SA_RESTART, sa_restorer=0x7f44490e46b0}, 8) = 0
close(4) = 0
close(5) = 0
exit_group(0) = ?
+++ exited with 0 +++
Second with the issue (i.e. wit exported variable SOMEVAR), again quoting only the lines as of loading Readline/Gnu library:
stat("/usr/lib64/perl5/vendor_perl/auto/Term/ReadLine/Gnu/XS/autosplit.ix", {st_mode=S_IFREG|0644, st_size=1137, ...}) = 0
openat(AT_FDCWD, "/usr/lib64/perl5/vendor_perl/auto/Term/ReadLine/Gnu/XS/autosplit.ix", O_RDONLY|O_CLOEXEC) = 13
ioctl(13, TCGETS, 0x7ffe8bf62e00) = -1 ENOTTY (Inappropriate ioctl for device)
lseek(13, 0, SEEK_CUR) = 0
read(13, "# Index created by AutoSplit for"..., 8192) = 1137
read(13, "", 8192) = 0
close(13) = 0
lseek(3, 2189, SEEK_SET) = 2189
lseek(3, 0, SEEK_CUR) = 2189
close(3) = 0
getuid() = 1000
geteuid() = 1000
getgid() = 1000
getegid() = 1000
fcntl(4, F_GETFL) = 0x8002 (flags O_RDWR|O_LARGEFILE)
fcntl(5, F_GETFL) = 0x8002 (flags O_RDWR|O_LARGEFILE)
ioctl(1, TCGETS, {B38400 opost isig icanon echo ...}) = 0
stat("/home/bernhard/.terminfo", 0x55b600a2b5a0) = -1 ENOENT (No such file or directory)
stat("/etc/terminfo", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
stat("/usr/share/terminfo", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
access("/etc/terminfo/x/xterm-256color", R_OK) = -1 ENOENT (No such file or directory)
access("/usr/share/terminfo/x/xterm-256color", R_OK) = 0
openat(AT_FDCWD, "/usr/share/terminfo/x/xterm-256color", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=3808, ...}) = 0
read(3, "\36\2%\0&\0\17\0\235\0010\6xterm-256color|xterm"..., 32768) = 3808
read(3, "", 28672) = 0
close(3) = 0
ioctl(1, TCGETS, {B38400 opost isig icanon echo ...}) = 0
ioctl(1, TCGETS, {B38400 opost isig icanon echo ...}) = 0
ioctl(1, TCGETS, {B38400 opost isig icanon echo ...}) = 0
ioctl(1, TCGETS, {B38400 opost isig icanon echo ...}) = 0
ioctl(1, TIOCGWINSZ, {ws_row=43, ws_col=173, ws_xpixel=0, ws_ypixel=0}) = 0
ioctl(4, TIOCGWINSZ, {ws_row=43, ws_col=173, ws_xpixel=0, ws_ypixel=0}) = 0
writev(2, [{iov_base="double free or corruption (!prev"..., iov_len=33}, {iov_base="\n", iov_len=1}], 2) = 34
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f376a2e3000
rt_sigprocmask(SIG_UNBLOCK, [ABRT], NULL, 8) = 0
rt_sigprocmask(SIG_BLOCK, ~[RTMIN RT_1], [], 8) = 0
getpid() = 4029
gettid() = 4029
tgkill(4029, 4029, SIGABRT) = 0
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
--- SIGABRT {si_signo=SIGABRT, si_code=SI_TKILL, si_pid=4029, si_uid=1000} ---
+++ killed by SIGABRT (core dumped) +++
PostgreSQL stuck on Recovery Mode (strace returns semop)
I have a Postgresql Server with many databases, one of the database got a stucked INSERT INTO and it hangs all the Database. After rebooting the server activates it´s recovery mode, i did a strace and got the following:
openat(AT_FDCWD, "base/14096409", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 755
getdents(755, /* 1024 entries */, 32768) = 32760
getdents(755, /* 1024 entries */, 32768) = 32768
getdents(755, /* 801 entries */, 32768) = 25632
getdents(755, /* 0 entries */, 32768) = 0
close(755) = 0
openat(AT_FDCWD, "base/14096409", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 755
getdents(755, /* 1024 entries */, 32768) = 32760
getdents(755, /* 1024 entries */, 32768) = 32768
getdents(755, /* 801 entries */, 32768) = 25632
getdents(755, /* 0 entries */, 32768) = 0
close(755) = 0
open("base/14096409", O_RDONLY) = 755
fsync(755) = 0
close(755) = 0
getdents(5, /* 0 entries */, 32768) = 0
close(5) = 0
openat(AT_FDCWD, "pg_tblspc", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 5
getdents(5, /* 2 entries */, 32768) = 48
getdents(5, /* 0 entries */, 32768) = 0
close(5) = 0
open("pg_xlog/000000010000004E0000008B", O_RDWR) = 5
openat(AT_FDCWD, "pg_twophase", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 755
getdents(755, /* 2 entries */, 32768) = 48
getdents(755, /* 0 entries */, 32768) = 0
close(755) = 0
open("base/13702059/13709821", O_RDWR) = 755
lseek(755, 16384, SEEK_SET) = 16384
lseek(755, 0, SEEK_END) = 32768
write(328, "M\0\0\0\270\372\206w\1\0\4\0008\0\360\1\0 \4 \0\0\0\0 \235\300\5\30\232\n\6"..., 8192) = 8192
lseek(755, 8192, SEEK_SET) = 8192
read(755, "N\0\0\0\20\31\224f\1\0\0\0\374\1\310\22\360\37\4 \0\0\0\0\310\2220\0\330\2370\0"..., 8192) = 8192
lseek(755, 0, SEEK_END) = 32768
write(328, "M\0\0\0\340\rQx\1\0\4\0008\0P\1\0 \4 \0\0\0\0(\235\256\5H\232\300\5"..., 8192) = 8192
lseek(755, 24576, SEEK_SET) = 24576
read(755, "N\0\0\0\20\31\224f\1\0\0\0X\0028\21\360\37\4 \0\0\0\0008\221#\0X\2210\0"..., 8192) = 8192
write(328, "M\0\0\0 [$y\1\0\4\0008\0X\1\0 \4 \0\0\0\0(\235\254\5H\232\274\5"..., 8192) = 8192
lseek(755, 0, SEEK_SET) = 0
read(755, "I\0\0\0\310\307\10\t\1\0\0\0000\0\360\37\360\37\4 \0\0\0\0b1\5\0\2\0\0\0"..., 8192) = 8192
semop(3637260, [{12, -1, 0}], 1
Any advice what can i do ? Thanks so much !
bind error when doing traceroute
I'm not very knowledgeable in named and bind, but after setting up my domain and playing with named (trying to set up my private email server) I ended up probably messing up something. Taceroute does not work anymore, I get a bind error (ping works). Relevant part of "strare traceroute xxx.xxx.xxx.xxx" is below. Can anybody please guide me to a solution - thx jankom
socket(PF_INET, SOCK_RAW, IPPROTO_ICMP) = 10
socket(PF_INET, SOCK_RAW, IPPROTO_RAW) = 11
getuid32() = 0
setuid32(0) = 0
getpid() = 3212
socket(PF_INET, SOCK_DGRAM, IPPROTO_IP) = 12
bind(12, {sa_family=AF_UNSPEC, sa_data="\0\0\0\0\0\0\0\0\0\0\0\0\0\0"}, 16) = -1 EINVAL (Invalid argument)
dup(2) = 13
fcntl64(13, F_GETFL) = 0x2 (flags O_RDWR)
fstat64(13, {st_mode=S_IFCHR|0620, st_rdev=makedev(136, 4), ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77c3000
_llseek(13, 0, 0xbfc4ebc0, SEEK_CUR) = -1 ESPIPE (Illegal seek)
write(13, "bind: Invalid argument\n", 23bind: Invalid argument) = 23
) 23
close(13) = 0
munmap(0xb77c3000, 4096) = 0
exit_group(1) = ?
Unable to set TCP_NODELAY with setsockopt on CentOS
The call to setsockopt seems to succeed, but checking with getsockopt shows that TCP_NODELAY has not been set properly.
Below is the relevant section of my strace.
Am I missing something?
[00007fd101b10327] socket(PF_INET, SOCK_STREAM, IPPROTO_TCP) = 4
[00007fd102c6d520] connect(4, {sa_family=AF_INET, sin_port=htons(31695), sin_addr=inet_addr("[removed]")}, 16) = 0
[00007fd102c6d1e0] write(3, "110816.344860 [I.COMM_NET] TCP c"..., 155) = 155
[00007fd101a4e880] rt_sigprocmask(SIG_BLOCK, [CHLD], [PIPE], 8) = 0
[00007fd101a4e75d] rt_sigaction(SIGCHLD, NULL, {SIG_DFL, [], 0}, 8) = 0
[00007fd101a4e880] rt_sigprocmask(SIG_SETMASK, [PIPE], NULL, 8) = 0
[00007fd101ad6470] nanosleep({1, 0}, 0x7fff66f9df40) = 0
[00007fd101b102ca] setsockopt(4, SOL_TCP, TCP_NODELAY, [1], 4) = 0
[00007fd101a4e880] rt_sigprocmask(SIG_BLOCK, [CHLD], [PIPE], 8) = 0
[00007fd101a4e75d] rt_sigaction(SIGCHLD, NULL, {SIG_DFL, [], 0}, 8) = 0
[00007fd101a4e880] rt_sigprocmask(SIG_SETMASK, [PIPE], NULL, 8) = 0
[00007fd101ad6470] nanosleep({1, 0}, 0x7fff66f9df40) = 0
[00007fd101b102ca] setsockopt(4, SOL_TCP, TCP_NODELAY, [1], 4) = 0
[00007fd101a4e880] rt_sigprocmask(SIG_BLOCK, [CHLD], [PIPE], 8) = 0
[00007fd101a4e75d] rt_sigaction(SIGCHLD, NULL, {SIG_DFL, [], 0}, 8) = 0
[00007fd101a4e880] rt_sigprocmask(SIG_SETMASK, [PIPE], NULL, 8) = 0
[00007fd101ad6470] nanosleep({1, 0}, 0x7fff66f9df40) = 0
[00007fd101b0ff8a] getsockopt(4, SOL_TCP, TCP_NODELAY, "", [0]) = 0
The strace output isn't telling you anything about the value of the TCP_NODELAY option. What it is telling you is that you are calling getsockopt with a zero-length buffer for the response. The kernel will not be able to return any useful information when the length of the buffer is zero. In the output [0] is an indication of the size of the buffer you gave it. You need to specify a buffer size which is at least the size of an integer in order to get a useful response back.
Why does IPC::SysV->shmget respond with EINVAL?
I am currently running perl 5.8.8 on a server and I'm trying to install 5.14.
I configured it to usethreads and use64bitint and otherwise the defaults it suggested.
make ran without problems, but make test is failing, on
../cpan/IPC-SysV/t/ipcsysv.t
../cpan/IPC-SysV/t/shm.t
thus:
# ./perl harness ../cpan/IPC-SysV/t/shm.t ../cpan/IPC-SysV/t/ipcsysv.t
../cpan/IPC-SysV/t/shm.t ...... IPC::SharedMem->new failed: Invalid argument at t/shm.t line 54.
../cpan/IPC-SysV/t/shm.t ...... Dubious, test returned 22 (wstat 5632, 0x1600)
No subtests run
../cpan/IPC-SysV/t/ipcsysv.t .. 1/38 shmget failed: Invalid argument at t/ipcsysv.t line 100.
# Looks like you planned 38 tests but ran 17.
# Looks like your test exited with 22 just after 17.
../cpan/IPC-SysV/t/ipcsysv.t .. Dubious, test returned 22 (wstat 5632, 0x1600)
Failed 21/38 subtests
Test Summary Report
-------------------
../cpan/IPC-SysV/t/shm.t (Wstat: 5632 Tests: 0 Failed: 0)
Non-zero exit status: 22
Parse errors: No plan found in TAP output
../cpan/IPC-SysV/t/ipcsysv.t (Wstat: 5632 Tests: 17 Failed: 0)
Non-zero exit status: 22
Parse errors: Bad plan. You planned 38 tests but ran 17.
Files=2, Tests=17, 0 wallclock secs ( 0.01 usr 0.00 sys + 0.13 cusr 0.00 csys = 0.14 CPU)
Result: FAIL
Both of these tests are reporting 'Invalid argument', but when I look at the source, I can't see anything that looks invalid. I'm not really sure how to proceed... any pointers?
UPDATE
I ran
strace perl -MIPC::SysV=IPC_PRIVATE,S_IRWXU -e 'shmget(IPC_PRIVATE, 8, S_IRWXU) or die $!'
on two servers: one which is having these problems and one which is not.
There was a lot of output, but what appears interesting is this:
GOOD:
.
.
.
stat64("/usr/lib/perl5/site_perl/5.8.8/i386-linux-thread-multi/auto/IPC/SysV", 0x9d7f0c8) = -1 ENOENT (No such file or directory)
stat64("/usr/lib/perl5/site_perl/5.8.8/auto/IPC/SysV", 0x9d7f0c8) = -1 ENOENT (No such file or directory)
stat64("/usr/lib/perl5/site_perl/auto/IPC/SysV", 0x9d7f0c8) = -1 ENOENT (No such file or directory)
stat64("/usr/lib/perl5/vendor_perl/5.8.8/i386-linux-thread-multi/auto/IPC/SysV", 0x9d7f0c8) = -1 ENOENT (No such file or directory)
stat64("/usr/lib/perl5/vendor_perl/5.8.8/auto/IPC/SysV", 0x9d7f0c8) = -1 ENOENT (No such file or directory)
stat64("/usr/lib/perl5/vendor_perl/auto/IPC/SysV", 0x9d7f0c8) = -1 ENOENT (No such file or directory)
stat64("/usr/lib/perl5/5.8.8/i386-linux-thread-multi/auto/IPC/SysV", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
stat64("/usr/lib/perl5/5.8.8/i386-linux-thread-multi/auto/IPC/SysV/SysV.so", {st_mode=S_IFREG|0755, st_size=15072, ...}) = 0
stat64("/usr/lib/perl5/5.8.8/i386-linux-thread-multi/auto/IPC/SysV/SysV.bs", 0x9d7f0c8) = -1 ENOENT (No such file or directory)
futex(0x4d106c, FUTEX_WAKE_PRIVATE, 2147483647) = 0
open("/usr/lib/perl5/5.8.8/i386-linux-thread-multi/auto/IPC/SysV/SysV.so", O_RDONLY) = 4
read(4, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0 \v\0\0004\0\0\0"..., 512) = 512
fstat64(4, {st_mode=S_IFREG|0755, st_size=15072, ...}) = 0
mmap2(NULL, 17948, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 4, 0) = 0x588000
mmap2(0x58c000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 4, 0x3) = 0x58c000
close(4) = 0
close(3) = 0
shmget(IPC_PRIVATE, 8, 0700) = 7438344
exit_group(0)
BAD:
.
.
.
stat64("/usr/lib/perl5/site_perl/5.8.8/i386-linux-thread-multi/auto/IPC/SysV", 0x8d290c8) = -1 ENOENT (No such file or directory)
stat64("/usr/lib/perl5/site_perl/5.8.8/auto/IPC/SysV", 0x8d290c8) = -1 ENOENT (No such file or directory)
stat64("/usr/lib/perl5/site_perl/auto/IPC/SysV", 0x8d290c8) = -1 ENOENT (No such file or directory)
stat64("/usr/lib/perl5/vendor_perl/5.8.8/i386-linux-thread-multi/auto/IPC/SysV", 0x8d290c8) = -1 ENOENT (No such file or directory)
stat64("/usr/lib/perl5/vendor_perl/5.8.8/auto/IPC/SysV", 0x8d290c8) = -1 ENOENT (No such file or directory)
stat64("/usr/lib/perl5/vendor_perl/auto/IPC/SysV", 0x8d290c8) = -1 ENOENT (No such file or directory)
stat64("/usr/lib/perl5/5.8.8/i386-linux-thread-multi/auto/IPC/SysV", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
stat64("/usr/lib/perl5/5.8.8/i386-linux-thread-multi/auto/IPC/SysV/SysV.so", {st_mode=S_IFREG|0755, st_size=15072, ...}) = 0
stat64("/usr/lib/perl5/5.8.8/i386-linux-thread-multi/auto/IPC/SysV/SysV.bs", 0x8d290c8) = -1 ENOENT (No such file or directory)
futex(0x94306c, FUTEX_WAKE_PRIVATE, 2147483647) = 0
open("/usr/lib/perl5/5.8.8/i386-linux-thread-multi/auto/IPC/SysV/SysV.so", O_RDONLY) = 4
read(4, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0 \v\0\0004\0\0\0"..., 512) = 512
fstat64(4, {st_mode=S_IFREG|0755, st_size=15072, ...}) = 0
mmap2(NULL, 17948, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 4, 0) = 0x6a4000
mmap2(0x6a8000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 4, 0x3) = 0x6a8000
close(4) = 0
close(3) = 0
shmget(IPC_PRIVATE, 8, 0700) = -1 EINVAL (Invalid argument)
open("/usr/share/locale/locale.alias", O_RDONLY) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=2528, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7dbe000
read(3, "# Locale name alias data base.\n#"..., 4096) = 2528
read(3, "", 4096) = 0
close(3) = 0
munmap(0xb7dbe000, 4096) = 0
open("/usr/share/locale/en_US.UTF-8/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en_US.utf8/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en_US/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en.UTF-8/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en.utf8/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
write(2, "Invalid argument at -e line 1.\n", 31Invalid argument at -e line 1.
) = 31
exit_group(22) = ?
So, it appears that the same thing is happening on both servers, it's just that on one, I see
shmget(IPC_PRIVATE, 8, 0700) = 7438344
and the other, I see
shmget(IPC_PRIVATE, 8, 0700) = -1 EINVAL (Invalid argument)
The versions of IPC::SysV are the same on both servers... but it looks to me that this isn't relevant, and that the problem is the the code making the system call... right?
What next?
** UPDATE 2 **
After some googling, I ran the following:
GOOD:
# cat /proc/sys/kernel/shmmax
4294967295
BAD:
# cat /proc/sys/kernel/shmmax
0
So, that explains the EINVAL, since (from the man pages)
EINVAL
A new segment was to be created and size < SHMMIN or size > SHMMAX, or no new segment
was to be created, a segment with given key existed, but size is greater than the size
of that segment.
Now, my question is, is there a good reason why this might be set at zero?
Problem solved. The /etc/sysctl.conf file contained the following: kernel.shmmax = 137438953472 This is a 64 bit value, but the system is a 32 bit system. As a result, the SHMMAX value was being set to 0, making all calls to shmget fail. Changing it to kernel.shmmax = 4294967295 And using echo 4294967295 >/proc/sys/kernel/shmmax I changed the value of SHMMAX and the test completed successfully.