failure to use libibverbs clone

[Or Gerlitz]

Roland,

Using a fresh clone of libibverbs, libmthca and a kernel based on 2.6.20-rc3 (clone of
Sean's rdma-dev git tree at open fabrics)

I am getting errors such as

# LD_LIBRARY_PATH=/usr/local/rdmacm/lib /usr/local/rdmacm/bin/ibv_devinfo

libibverbs: Warning: couldn't open config directory '/usr/local/rdmacm/etc/libibverbs.d'.
libibverbs: Warning: no userspace device-specific driver found for /sys/class/infiniband_verbs/uverbs0
No IB devices found

the strace traces follow, the system is very much operative (eg with IPoIB)

Or.


execve("/usr/local/rdmacm/bin/ibv_devinfo", ["/usr/local/rdmacm/bin/ibv_devinfo"], [/* 70 vars */]) = 0
uname({sys="Linux", node="dill", ...}) = 0
brk(0) = 0x503000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2aeba2f6f000
open("/etc/ld.so.preload", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/rdmacm/lib/tls/x86_64/libibverbs.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/rdmacm/lib/tls/x86_64", 0x7fff07b4e0f0) = -1 ENOENT (No such file or directory)
open("/usr/local/rdmacm/lib/tls/libibverbs.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/rdmacm/lib/tls", 0x7fff07b4e0f0) = -1 ENOENT (No such file or directory)
open("/usr/local/rdmacm/lib/x86_64/libibverbs.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/usr/local/rdmacm/lib/x86_64", 0x7fff07b4e0f0) = -1 ENOENT (No such file or directory)
open("/usr/local/rdmacm/lib/libibverbs.so.2", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320\'\0"..., 640) = 640
fstat(3, {st_mode=S_IFREG|0755, st_size=164431, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2aeba2f70000
mmap(NULL, 1085352, PROT_READ|PROT_EXEC, MAP_PRIVATE, 3, 0) = 0x2aeba3071000
madvise(0x2aeba3071000, 1085352, MADV_SEQUENTIAL|0x1) = 0
mprotect(0x2aeba3079000, 1052584, PROT_NONE) = 0
mmap(0x2aeba3171000, 36864, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 3, 0) = 0x2aeba3171000
close(3) = 0
open("/usr/local/rdmacm/lib/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/rdmacm/lib/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=130091, ...}) = 0
mmap(NULL, 130091, PROT_READ, MAP_PRIVATE, 3, 0) = 0x2aeba317a000
close(3) = 0
open("/lib64/tls/libpthread.so.0", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\340X\0\0"..., 640) = 640
fstat(3, {st_mode=S_IFREG|0755, st_size=99188, ...}) = 0
mmap(NULL, 1129880, PROT_READ|PROT_EXEC, MAP_PRIVATE, 3, 0) = 0x2aeba319a000
madvise(0x2aeba319a000, 1129880, MADV_SEQUENTIAL|0x1) = 0
mprotect(0x2aeba31a8000, 1072536, PROT_NONE) = 0
mmap(0x2aeba329a000, 65536, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 3, 0) = 0x2aeba329a000
mmap(0x2aeba32aa000, 15768, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x2aeba32aa000
close(3) = 0
open("/usr/local/rdmacm/lib/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/rdmacm/lib/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/lib64/libdl.so.2", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320\36\0"..., 640) = 640
fstat(3, {st_mode=S_IFREG|0755, st_size=16807, ...}) = 0
mmap(NULL, 1058904, PROT_READ|PROT_EXEC, MAP_PRIVATE, 3, 0) = 0x2aeba32ae000
madvise(0x2aeba32ae000, 1058904, MADV_SEQUENTIAL|0x1) = 0
mprotect(0x2aeba32b1000, 1046616, PROT_NONE) = 0
mmap(0x2aeba33ae000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 3, 0) = 0x2aeba33ae000
close(3) = 0
open("/usr/local/rdmacm/lib/libc.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/local/rdmacm/lib/libc.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/lib64/tls/libc.so.6", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0p\313\1\0"..., 640) = 640
lseek(3, 624, SEEK_SET) = 624
read(3, "\4\0\0\0\20\0\0\0\1\0\0\0GNU\0\0\0\0\0\2\0\0\0\6\0\0\0"..., 32) = 32
fstat(3, {st_mode=S_IFREG|0755, st_size=1401317, ...}) = 0
mmap(NULL, 2235432, PROT_READ|PROT_EXEC, MAP_PRIVATE, 3, 0) = 0x2aeba33b1000
madvise(0x2aeba33b1000, 2235432, MADV_SEQUENTIAL|0x1) = 0
mprotect(0x2aeba34b7000, 1162280, PROT_NONE) = 0
mmap(0x2aeba35b1000, 122880, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 3, 0x100000) = 0x2aeba35b1000
mmap(0x2aeba35cf000, 15400, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x2aeba35cf000
close(3) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2aeba35d3000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2aeba35d4000
arch_prctl(0x1002, 0x2aeba35d3900) = 0
munmap(0x2aeba317a000, 130091) = 0
SYS_218(0x2aeba35d3990, 0x7fff07b4ecc8, 0x2aeba35d3900, 0xffffffffffffffff, 0xffffffff00000000) = 8760
rt_sigaction(SIGRTMIN, {0x2aeba319f840, [], SA_SIGINFO|0x4000000}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN], NULL, 8) = 0
getrlimit(0x3, 0x7fff07b4eb50) = 0
_sysctl({{CTL_KERN, KERN_VERSION}, 2, 0x7fff07b4e950, 35, (nil), 0}) = 0
getuid() = 0
geteuid() = 0
brk(0) = 0x503000
brk(0x524000) = 0x524000
brk(0) = 0x524000
open("/sys/class/infiniband_verbs/abi_version", O_RDONLY) = 3
read(3, "6\n", 8) = 2
close(3) = 0
open("/usr/local/rdmacm/etc/libibverbs.d", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = -1 ENOENT (No such file or directory)
write(2, "libibverbs: Warning: couldn\'t op"..., 90libibverbs: Warning: couldn't open config directory '/usr/local/rdmacm/etc/libibverbs.d'.
) = 90
open("/sys/class/infiniband_verbs", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = 3
fstat(3, {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
fcntl(3, F_SETFD, FD_CLOEXEC) = 0
getdents(3, /* 4 entries */, 4096) = 112
stat("/sys/class/infiniband_verbs/uverbs0", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
open("/sys/class/infiniband_verbs/uverbs0/ibdev", O_RDONLY) = 4
read(4, "mthca0\n", 64) = 7
close(4) = 0
open("/sys/class/infiniband_verbs/uverbs0/abi_version", O_RDONLY) = 4
read(4, "1\n", 8) = 2
close(4) = 0
stat("/sys/class/infiniband_verbs/abi_version", {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
getdents(3, /* 0 entries */, 4096) = 0
close(3) = 0
futex(0x2aeba33b0854, FUTEX_WAKE, 2147483647) = 0
getuid() = 0
geteuid() = 0
write(2, "libibverbs: Warning: no userspac"..., 103libibverbs: Warning: no userspace device-specific driver found for /sys/class/infiniband_verbs/uverbs0
) = 103
write(2, "No IB devices found\n", 20No IB devices found
) = 20
exit_group(-1) = ?

[Michael S. Tsirkin]

Post by Or Gerlitz
libibverbs: Warning: couldn't open config directory '/usr/local/rdmacm/etc/libibverbs.d'.

Well, do you have /usr/local/rdmacm/etc/libibverbs.d?

[Or Gerlitz]

Post by Michael S. Tsirkin

Post by Or Gerlitz
libibverbs: Warning: couldn't open config directory '/usr/local/rdmacm/etc/libibverbs.d'.

Well, do you have /usr/local/rdmacm/etc/libibverbs.d?

no, who should create it? doing $ make install under libibverbs does not
do it. I have created manually an empty library and then this warning
went away but the other one and the failure to find devices stayed.

Or.

[Michael S. Tsirkin]

Subject: Re: failure to use libibverbs clone

Post by Michael S. Tsirkin

Post by Or Gerlitz
libibverbs: Warning: couldn't open config directory '/usr/local/rdmacm/etc/libibverbs.d'.

Well, do you have /usr/local/rdmacm/etc/libibverbs.d?

no, who should create it? doing $ make install under libibverbs does not
do it. I have created manually an empty library and then this warning
went away but the other one and the failure to find devices stayed.

It's created when you install a low level driver (e.g. libmthca).

[Or Gerlitz]

install libmthca does not seem to create the etc directory

Or.


dill:/usr/src/libmthca # make install
make[1]: Entering directory `/usr/src/libmthca'
test -z "/usr/local/rdmacm/lib" || mkdir -p -- . "/usr/local/rdmacm/lib"
test -z "" || mkdir -p -- . ""
test -z "/usr/local/rdmacm/lib/infiniband" || mkdir -p -- .
"/usr/local/rdmacm/lib/infiniband"
/bin/sh ./libtool --mode=install /usr/bin/install -c 'src/mthca.la'
'/usr/local/rdmacm/lib/infiniband/mthca.la'
/usr/bin/install -c src/.libs/mthca.so
/usr/local/rdmacm/lib/infiniband/mthca.so
/usr/bin/install -c src/.libs/mthca.lai
/usr/local/rdmacm/lib/infiniband/mthca.la
/usr/bin/install -c src/.libs/mthca.a
/usr/local/rdmacm/lib/infiniband/mthca.a
ranlib /usr/local/rdmacm/lib/infiniband/mthca.a
chmod 644 /usr/local/rdmacm/lib/infiniband/mthca.a
PATH="$PATH:/sbin" ldconfig -n /usr/local/rdmacm/lib/infiniband
----------------------------------------------------------------------
Libraries have been installed in:
/usr/local/rdmacm/lib/infiniband

If you ever happen to want to link against installed libraries
in a given directory, LIBDIR, you must either use libtool, and
specify the full pathname of the library, or use the `-LLIBDIR'
flag during linking and do at least one of the following:
- add LIBDIR to the `LD_LIBRARY_PATH' environment variable
during execution
- add LIBDIR to the `LD_RUN_PATH' environment variable
during linking
- use the `-Wl,--rpath -Wl,LIBDIR' linker flag
- have your system administrator add LIBDIR to `/etc/ld.so.conf'

See any operating system documentation about shared libraries for
more information, such as the ld(1) and ld.so(8) manual pages.
----------------------------------------------------------------------
make[1]: Leaving directory `/usr/src/libmthca'

[Michael S. Tsirkin]

Subject: Re: failure to use libibverbs clone
install libmthca does not seem to create the etc directory
Or.
dill:/usr/src/libmthca # make install
make[1]: Entering directory `/usr/src/libmthca'
test -z "/usr/local/rdmacm/lib" || mkdir -p -- . "/usr/local/rdmacm/lib"
test -z "" || mkdir -p -- . ""
test -z "/usr/local/rdmacm/lib/infiniband" || mkdir -p -- .
"/usr/local/rdmacm/lib/infiniband"
/bin/sh ./libtool --mode=install /usr/bin/install -c 'src/mthca.la'
'/usr/local/rdmacm/lib/infiniband/mthca.la'
/usr/bin/install -c src/.libs/mthca.so
/usr/local/rdmacm/lib/infiniband/mthca.so
/usr/bin/install -c src/.libs/mthca.lai
/usr/local/rdmacm/lib/infiniband/mthca.la
/usr/bin/install -c src/.libs/mthca.a
/usr/local/rdmacm/lib/infiniband/mthca.a
ranlib /usr/local/rdmacm/lib/infiniband/mthca.a
chmod 644 /usr/local/rdmacm/lib/infiniband/mthca.a
PATH="$PATH:/sbin" ldconfig -n /usr/local/rdmacm/lib/infiniband
----------------------------------------------------------------------
/usr/local/rdmacm/lib/infiniband

This is your problem right here.
You have configured mthca against an old libibverbs installation,
so it puts libraries in the wrong places.

[Or Gerlitz]

I have configured libmthca against the install of libibverbs (/usr/local/rdmacm),
the configure output is below.

Or.

dill:/usr/src/libmthca # ./configure --prefix=/usr/local/rdmacm CFLAGS=-I/usr/local/rdmacm/include \
LDFLAGS=-L/usr/local/rdmacm/lib LD_LIBRARY_PATH=/usr/local/rdmacm/lib

checking for a BSD-compatible install... /usr/bin/install -c
checking whether build environment is sane... yes
checking for gawk... gawk
checking whether make sets $(MAKE)... yes
checking build system type... x86_64-suse-linux
checking host system type... x86_64-suse-linux
checking for style of include used by make... GNU
checking for gcc... gcc
checking for C compiler default output file name... a.out
checking whether the C compiler works... yes
checking whether we are cross compiling... no
checking for suffix of executables...
checking for suffix of object files... o
checking whether we are using the GNU C compiler... yes
checking whether gcc accepts -g... yes
checking for gcc option to accept ANSI C... none needed
checking dependency style of gcc... gcc3
checking for a sed that does not truncate output... /usr/bin/sed
checking for egrep... grep -E
checking for ld used by gcc... /usr/x86_64-suse-linux/bin/ld
checking if the linker (/usr/x86_64-suse-linux/bin/ld) is GNU ld... yes
checking for /usr/x86_64-suse-linux/bin/ld option to reload object
files... -r
checking for BSD-compatible nm... /usr/bin/nm -B
checking whether ln -s works... yes
checking how to recognise dependent libraries... pass_all
checking how to run the C preprocessor... gcc -E
checking for ANSI C header files... yes
checking for sys/types.h... yes
checking for sys/stat.h... yes
checking for stdlib.h... yes
checking for string.h... yes
checking for memory.h... yes
checking for strings.h... yes
checking for inttypes.h... yes
checking for stdint.h... yes
checking for unistd.h... yes
checking dlfcn.h usability... yes
checking dlfcn.h presence... yes
checking for dlfcn.h... yes
checking for g++... g++
checking whether we are using the GNU C++ compiler... yes
checking whether g++ accepts -g... yes
checking dependency style of g++... gcc3
checking how to run the C++ preprocessor... g++ -E
checking for g77... g77
checking whether we are using the GNU Fortran 77 compiler... yes
checking whether g77 accepts -g... yes
checking the maximum length of command line arguments... 32768
checking command to parse /usr/bin/nm -B output from gcc object... ok
checking for objdir... .libs
checking for ar... ar
checking for ranlib... ranlib
checking for strip... strip
checking if gcc static flag works... yes
checking if gcc supports -fno-rtti -fno-exceptions... no
checking for gcc option to produce PIC... -fPIC
checking if gcc PIC flag -fPIC works... yes
checking if gcc supports -c -o file.o... yes
checking whether the gcc linker (/usr/x86_64-suse-linux/bin/ld -m
elf_x86_64) supports shared libraries... yes
checking whether -lc should be explicitly linked in... no
checking dynamic linker characteristics... GNU/Linux ld.so
checking how to hardcode library paths into programs... immediate
checking whether stripping libraries is possible... yes
checking if libtool supports shared libraries... yes
checking whether to build shared libraries... yes
checking whether to build static libraries... yes
configure: creating libtool
appending configuration tag "CXX" to libtool
checking for ld used by g++... /usr/x86_64-suse-linux/bin/ld -m elf_x86_64
checking if the linker (/usr/x86_64-suse-linux/bin/ld -m elf_x86_64) is
GNU ld... yes
checking whether the g++ linker (/usr/x86_64-suse-linux/bin/ld -m
elf_x86_64) supports shared libraries... yes
checking for g++ option to produce PIC... -fPIC
checking if g++ PIC flag -fPIC works... yes
checking if g++ supports -c -o file.o... yes
checking whether the g++ linker (/usr/x86_64-suse-linux/bin/ld -m
elf_x86_64) supports shared libraries... yes
checking dynamic linker characteristics... GNU/Linux ld.so
checking how to hardcode library paths into programs... immediate
checking whether stripping libraries is possible... yes
appending configuration tag "F77" to libtool
checking if libtool supports shared libraries... yes
checking whether to build shared libraries... yes
checking whether to build static libraries... yes
checking for g77 option to produce PIC... -fPIC
checking if g77 PIC flag -fPIC works... yes
checking if g77 supports -c -o file.o... yes
checking whether the g77 linker (/usr/x86_64-suse-linux/bin/ld -m
elf_x86_64) supports shared libraries... yes
checking dynamic linker characteristics... GNU/Linux ld.so
checking how to hardcode library paths into programs... immediate
checking whether stripping libraries is possible... yes
checking for gcc... (cached) gcc
checking whether we are using the GNU C compiler... (cached) yes
checking whether gcc accepts -g... (cached) yes
checking for gcc option to accept ANSI C... (cached) none needed
checking dependency style of gcc... (cached) gcc3
checking for ibv_get_device_list in -libverbs... yes
checking infiniband/driver.h usability... yes
checking infiniband/driver.h presence... no
configure: WARNING: infiniband/driver.h: accepted by the compiler, rejected by the preprocessor!
configure: WARNING: infiniband/driver.h: proceeding with the compiler's result
checking for infiniband/driver.h... yes
checking for ANSI C header files... (cached) yes
checking valgrind/memcheck.h usability... no
checking valgrind/memcheck.h presence... no
checking for valgrind/memcheck.h... no
checking for an ANSI C-conforming const... yes
checking for long... yes
checking size of long... 8
checking for ibv_read_sysfs_file... yes
checking for ibv_dontfork_range... yes
checking for ibv_dofork_range... yes
checking for ibv_register_driver... yes
checking whether ld accepts --version-script...
configure: creating ./config.status
config.status: creating Makefile
config.status: creating libmthca.spec
config.status: creating config.h
config.status: config.h is unchanged
config.status: executing depfiles commands

[Michael S. Tsirkin]

Subject: Re: failure to use libibverbs clone
I have configured libmthca against the install of libibverbs (/usr/local/rdmacm),
the configure output is below.
Or.
dill:/usr/src/libmthca # ./configure --prefix=/usr/local/rdmacm CFLAGS=-I/usr/local/rdmacm/include \
LDFLAGS=-L/usr/local/rdmacm/lib LD_LIBRARY_PATH=/usr/local/rdmacm/lib

Works for me.
Look at how configure defines HAVE_IBV_DEVICE_LIBRARY_EXTENSION,
this is what goes wrong for you.

configure: WARNING: infiniband/driver.h: accepted by the compiler, rejected by the preprocessor!
configure: WARNING: infiniband/driver.h: proceeding with the compiler's result

Try also checking why do you see this.