Bug#919447: [asterisk] [1329393.987739] asterisk[27758]: segfault at 10 ip 00007facf79d5233 sp 00007facc14428d0 error 4 in libiksemel.so.3.1.1[7facf79ce000+d000]
Bernhard Übelacker
bernhardu at mailbox.org
Wed Jan 16 13:50:28 GMT 2019
Hello Fernando Toledo, Dear Maintainer,
On Wed, 16 Jan 2019 01:22:31 -0300 Fernando Toledo <ftoledo at docksud.com.ar> wrote:
> > I need help to know how to get more info/debug.
Maybe you can install a core file collector.
On stable or testing I would propose systemd-coredump,
but that is not available in jessie.
The package corekeeper sounds similar and might put
after a crash a core file somewhere below /var/crash.
That might provide some more information when opened with gdb:
gdb -q /usr/sbin/asterisk /var/crash/<corefile>
backtrace
The other way would be to install gdb and attach that to the
live process. This maybe in a detachable session like in tmux.
tmux
gdb -q -ex 'set width' -ex 'set pagination off' -ex 'cont' -ex 'backtrace' -ex 'generate-core-file ~/asterisk.core' -ex 'detach' -ex 'quit' --pid $(pidof asterisk)
But without the debug information that might not be sufficient.
In that case rebuilding the packages with debug
information may be needed. See pointer for that in [1].
> > i found that my asterisk crash in dmesg on stretch:
> >
> > [1329393.987739] asterisk[27758]: segfault at 10 ip 00007facf79d5233 sp
> > 00007facc14428d0 error 4 in libiksemel.so.3.1.1[7facf79ce000+d000]
That line may point to below line in libiksemel-1.4 in src/stream.c:552.
Kind regards,
Bernhard
[1] https://wiki.debian.org/HowToGetABacktrace
(gdb) disassemble /m iks_send_raw
Dump of assembler code for function iks_send_raw:
...
0x00007ffff7bd41f7 <+55>: js 0x7ffff7bd421b <iks_send_raw+91>
549 } else
550 #endif
551 {
552 ret = data->trans->send (data->sock, xmlstr, strlen (xmlstr));
0x00007ffff7bd4220 <+96>: callq 0x7ffff7bcfef0 <strlen at plt>
0x00007ffff7bd4225 <+101>: mov 0x10(%rbx),%rcx
0x00007ffff7bd4229 <+105>: mov 0x50(%rbx),%rdi
0x00007ffff7bd422d <+109>: mov %rax,%rdx
0x00007ffff7bd4230 <+112>: mov %rbp,%rsi
0x00007ffff7bd4233 <+115>: callq *0x10(%rcx) <<<<<<<<<<<<<< here $rcx seems to contain 0 -> data->trans == NULL ?
553 if (ret) return ret;
0x00007ffff7bd4236 <+118>: test %eax,%eax
0x00007ffff7bd4238 <+120>: jne 0x7ffff7bd421b <iks_send_raw+91>
554 }
555 if (data->logHook) data->logHook (data->user_data, xmlstr, strlen (xmlstr), 0);
-------------- next part --------------
[1329393.987739] asterisk[27758]: segfault at 10 ip 00007facf79d5233 sp 00007facc14428d0 error 4 in libiksemel.so.3.1.1[7facf79ce000+d000]
https://stackoverflow.com/questions/2549214/interpreting-segfault-messages
"error 4" == 0b100
/*
* Page fault error code bits:
*
* bit 0 == 0: no page found 1: protection fault
* bit 1 == 0: read access 1: write access
* bit 2 == 0: kernel-mode access 1: user-mode access
* bit 3 == 1: use of reserved bit detected
* bit 4 == 1: fault was an instruction fetch
*/
-->
0: no page found
0: read access
1: user-mode access
############
# Jessie amd64 qemu VM 2019-01-16
apt update
apt dist-upgrade
apt install corekeeper gdb binutils asterisk
# objdump --disassemble /usr/lib/x86_64-linux-gnu/libiksemel.so.3 | grep 233
7233: ff 51 10 callq *0x10(%rcx) --> likely -> offset of 0x10 matches also the sefault line.
9233: 89 f1 mov %esi,%ecx --> unlikely
root at debian:~# objdump --disassemble /usr/lib/x86_64-linux-gnu/libiksemel.so.3 | grep 7233: -B 40 -A 4
00000000000071c0 <iks_send_raw>:
71c0: 41 54 push %r12
71c2: 55 push %rbp
71c3: 48 89 f5 mov %rsi,%rbp
71c6: 53 push %rbx
71c7: e8 a4 be ff ff callq 3070 <iks_user_data at plt>
71cc: f6 40 58 04 testb $0x4,0x58(%rax)
71d0: 48 89 c3 mov %rax,%rbx
71d3: 48 89 ef mov %rbp,%rdi
71d6: 74 48 je 7220 <iks_send_raw+0x60>
71d8: e8 13 bd ff ff callq 2ef0 <strlen at plt>
71dd: 48 8b 7b 70 mov 0x70(%rbx),%rdi
71e1: 48 89 c2 mov %rax,%rdx
71e4: 48 89 ee mov %rbp,%rsi
71e7: e8 f4 be ff ff callq 30e0 <gnutls_record_send at plt>
71ec: 48 89 c2 mov %rax,%rdx
71ef: b8 07 00 00 00 mov $0x7,%eax
71f4: 48 85 d2 test %rdx,%rdx
71f7: 78 22 js 721b <iks_send_raw+0x5b>
71f9: 4c 8b 63 38 mov 0x38(%rbx),%r12
71fd: 4d 85 e4 test %r12,%r12
7200: 74 17 je 7219 <iks_send_raw+0x59>
7202: 48 89 ef mov %rbp,%rdi
7205: e8 e6 bc ff ff callq 2ef0 <strlen at plt>
720a: 48 8b 7b 20 mov 0x20(%rbx),%rdi
720e: 31 c9 xor %ecx,%ecx
7210: 48 89 c2 mov %rax,%rdx
7213: 48 89 ee mov %rbp,%rsi
7216: 41 ff d4 callq *%r12
7219: 31 c0 xor %eax,%eax
721b: 5b pop %rbx
721c: 5d pop %rbp
721d: 41 5c pop %r12
721f: c3 retq
7220: e8 cb bc ff ff callq 2ef0 <strlen at plt>
7225: 48 8b 4b 10 mov 0x10(%rbx),%rcx
7229: 48 8b 7b 50 mov 0x50(%rbx),%rdi
722d: 48 89 c2 mov %rax,%rdx
7230: 48 89 ee mov %rbp,%rsi
7233: ff 51 10 callq *0x10(%rcx)
7236: 85 c0 test %eax,%eax
7238: 75 e1 jne 721b <iks_send_raw+0x5b>
723a: 4c 8b 63 38 mov 0x38(%rbx),%r12
723e: 4d 85 e4 test %r12,%r12
.
apt install dpkg-dev devscripts
apt-get build-dep libiksemel3
mkdir source/libiksemel3/orig -p
cd source/libiksemel3/orig
apt-get source libiksemel3
cd
cd source/libiksemel3
cp orig/ try1 -a
cd try1/libiksemel-1.4/
dpkg-buildpackage
find -iname "libiksemel.so.3*" -type f| xargs file
./debian/tmp/usr/lib/x86_64-linux-gnu/libiksemel.so.3.1.1: ELF 64-bit LSB shared object, x86-64, version 1 (SYSV), dynamically linked, BuildID[sha1]=784779ec1b89ef8bd25b3f24393bd92cdae04e21, not stripped
./debian/libiksemel3/usr/lib/x86_64-linux-gnu/libiksemel.so.3.1.1: ELF 64-bit LSB shared object, x86-64, version 1 (SYSV), dynamically linked, BuildID[sha1]=784779ec1b89ef8bd25b3f24393bd92cdae04e21, stripped
./src/.libs/libiksemel.so.3.1.1: ELF 64-bit LSB shared object, x86-64, version 1 (SYSV), dynamically linked, BuildID[sha1]=784779ec1b89ef8bd25b3f24393bd92cdae04e21, not stripped
cmp debian/libiksemel3/usr/lib/x86_64-linux-gnu/libiksemel.so.3.1.1 /usr/lib/x86_64-linux-gnu/libiksemel.so.3
debian/libiksemel3/usr/lib/x86_64-linux-gnu/libiksemel.so.3.1.1 /usr/lib/x86_64-linux-gnu/libiksemel.so.3 differieren: Byte 473, Zeile 1
# not yet reproducible? different compiler?
objdump --disassemble src/.libs/libiksemel.so.3.1.1 | grep "<iks_send_raw>" -A40
00000000000071c0 <iks_send_raw>:
71c0: 41 54 push %r12
71c2: 55 push %rbp
71c3: 48 89 f5 mov %rsi,%rbp
71c6: 53 push %rbx
71c7: e8 a4 be ff ff callq 3070 <iks_user_data at plt>
71cc: f6 40 58 04 testb $0x4,0x58(%rax)
71d0: 48 89 c3 mov %rax,%rbx
71d3: 48 89 ef mov %rbp,%rdi
71d6: 74 48 je 7220 <iks_send_raw+0x60>
71d8: e8 13 bd ff ff callq 2ef0 <strlen at plt>
71dd: 48 8b 7b 70 mov 0x70(%rbx),%rdi
71e1: 48 89 c2 mov %rax,%rdx
71e4: 48 89 ee mov %rbp,%rsi
71e7: e8 f4 be ff ff callq 30e0 <gnutls_record_send at plt>
71ec: 48 89 c2 mov %rax,%rdx
71ef: b8 07 00 00 00 mov $0x7,%eax
71f4: 48 85 d2 test %rdx,%rdx
71f7: 78 22 js 721b <iks_send_raw+0x5b>
71f9: 4c 8b 63 38 mov 0x38(%rbx),%r12
71fd: 4d 85 e4 test %r12,%r12
7200: 74 17 je 7219 <iks_send_raw+0x59>
7202: 48 89 ef mov %rbp,%rdi
7205: e8 e6 bc ff ff callq 2ef0 <strlen at plt>
720a: 48 8b 7b 20 mov 0x20(%rbx),%rdi
720e: 31 c9 xor %ecx,%ecx
7210: 48 89 c2 mov %rax,%rdx
7213: 48 89 ee mov %rbp,%rsi
7216: 41 ff d4 callq *%r12
7219: 31 c0 xor %eax,%eax
721b: 5b pop %rbx
721c: 5d pop %rbp
721d: 41 5c pop %r12
721f: c3 retq
7220: e8 cb bc ff ff callq 2ef0 <strlen at plt>
7225: 48 8b 4b 10 mov 0x10(%rbx),%rcx
7229: 48 8b 7b 50 mov 0x50(%rbx),%rdi
722d: 48 89 c2 mov %rax,%rdx
7230: 48 89 ee mov %rbp,%rsi
7233: ff 51 10 callq *0x10(%rcx)
7236: 85 c0 test %eax,%eax
.
-> This function looks quite similar, even the offsets
LD_PRELOAD=/home/benutzer/source/libiksemel3/try1/libiksemel-1.4/src/.libs/libiksemel.so.3.1.1 gdb -q -ex 'set width 0' -ex 'set pagination off' --args /usr/sbin/asterisk
Reading symbols from /usr/sbin/asterisk...(no debugging symbols found)...done.
(gdb) info target
Symbols from "/usr/sbin/asterisk".
Local exec file:
`/usr/sbin/asterisk', file type elf64-x86-64.
Entry point: 0x31e43
0x0000000000000238 - 0x0000000000000254 is .interp
...
0x00000000003f2e40 - 0x000000000042ccc0 is .bss
(gdb) b *0x31e43
Breakpoint 1 at 0x31e43
(gdb) run
Starting program: /usr/sbin/asterisk
Warning:
Cannot insert breakpoint 1.
Cannot access memory at address 0x31e43
(gdb) dele 1
(gdb) info target
Symbols from "/usr/sbin/asterisk".
Unix child process:
Using the running image of child process 15000.
While running this, GDB does not access memory from...
Local exec file:
`/usr/sbin/asterisk', file type elf64-x86-64.
Entry point: 0x555555585e43
0x0000555555554238 - 0x0000555555554254 is .interp
...
0x00007ffff7ffe000 - 0x00007ffff7ffe1b0 is .bss in /lib64/ld-linux-x86-64.so.2
(gdb) b *0x555555585e43
Breakpoint 2 at 0x555555585e43
(gdb) cont
Continuing.
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Breakpoint 2, 0x0000555555585e43 in ?? ()
(gdb) info share
From To Syms Read Shared Object Library
0x00007ffff7ddcb00 0x00007ffff7df5370 Yes /lib64/ld-linux-x86-64.so.2
No linux-vdso.so.1
0x00007ffff7bd0460 0x00007ffff7bd731b Yes /home/benutzer/source/libiksemel3/try1/libiksemel-1.4/src/.libs/libiksemel.so.3.1.1
0x00007ffff78414a0 0x00007ffff796d943 Yes /lib/x86_64-linux-gnu/libc.so.6
...
0x00007ffff43358b0 0x00007ffff433a1bc Yes (*) /usr/lib/x86_64-linux-gnu/libffi.so.6
(*): Shared library is missing debugging information.
(gdb) disassemble /m iks_send_raw
Dump of assembler code for function iks_send_raw:
542 {
0x00007ffff7bd41c0 <+0>: push %r12
0x00007ffff7bd41c2 <+2>: push %rbp
0x00007ffff7bd41c3 <+3>: mov %rsi,%rbp
0x00007ffff7bd41c6 <+6>: push %rbx
543 struct stream_data *data = iks_user_data (prs);
0x00007ffff7bd41c7 <+7>: callq 0x7ffff7bd0070 <iks_user_data at plt>
0x00007ffff7bd41d0 <+16>: mov %rax,%rbx
544 int ret;
545
546 #ifdef HAVE_GNUTLS
547 if (data->flags & SF_SECURE) {
0x00007ffff7bd41cc <+12>: testb $0x4,0x58(%rax)
0x00007ffff7bd41d6 <+22>: je 0x7ffff7bd4220 <iks_send_raw+96>
548 if (gnutls_record_send (data->sess, xmlstr, strlen (xmlstr)) < 0) return IKS_NET_RWERR;
0x00007ffff7bd41d3 <+19>: mov %rbp,%rdi
0x00007ffff7bd41d8 <+24>: callq 0x7ffff7bcfef0 <strlen at plt>
0x00007ffff7bd41dd <+29>: mov 0x70(%rbx),%rdi
0x00007ffff7bd41e1 <+33>: mov %rax,%rdx
0x00007ffff7bd41e4 <+36>: mov %rbp,%rsi
0x00007ffff7bd41e7 <+39>: callq 0x7ffff7bd00e0 <gnutls_record_send at plt>
0x00007ffff7bd41ec <+44>: mov %rax,%rdx
0x00007ffff7bd41ef <+47>: mov $0x7,%eax
0x00007ffff7bd41f4 <+52>: test %rdx,%rdx
0x00007ffff7bd41f7 <+55>: js 0x7ffff7bd421b <iks_send_raw+91>
549 } else
550 #endif
551 {
552 ret = data->trans->send (data->sock, xmlstr, strlen (xmlstr));
0x00007ffff7bd4220 <+96>: callq 0x7ffff7bcfef0 <strlen at plt>
0x00007ffff7bd4225 <+101>: mov 0x10(%rbx),%rcx
0x00007ffff7bd4229 <+105>: mov 0x50(%rbx),%rdi
0x00007ffff7bd422d <+109>: mov %rax,%rdx
0x00007ffff7bd4230 <+112>: mov %rbp,%rsi
0x00007ffff7bd4233 <+115>: callq *0x10(%rcx) <<<<<<<<<<<<<< here $rcx seems to contain 0 -> data->trans == NULL ?
553 if (ret) return ret;
0x00007ffff7bd4236 <+118>: test %eax,%eax
0x00007ffff7bd4238 <+120>: jne 0x7ffff7bd421b <iks_send_raw+91>
554 }
555 if (data->logHook) data->logHook (data->user_data, xmlstr, strlen (xmlstr), 0);
0x00007ffff7bd41f9 <+57>: mov 0x38(%rbx),%r12
0x00007ffff7bd41fd <+61>: test %r12,%r12
0x00007ffff7bd4200 <+64>: je 0x7ffff7bd4219 <iks_send_raw+89>
0x00007ffff7bd4202 <+66>: mov %rbp,%rdi
0x00007ffff7bd4205 <+69>: callq 0x7ffff7bcfef0 <strlen at plt>
0x00007ffff7bd420a <+74>: mov 0x20(%rbx),%rdi
0x00007ffff7bd420e <+78>: xor %ecx,%ecx
0x00007ffff7bd4210 <+80>: mov %rax,%rdx
0x00007ffff7bd4213 <+83>: mov %rbp,%rsi
0x00007ffff7bd4216 <+86>: callq *%r12
0x00007ffff7bd423a <+122>: mov 0x38(%rbx),%r12
0x00007ffff7bd423e <+126>: test %r12,%r12
0x00007ffff7bd4241 <+129>: jne 0x7ffff7bd4202 <iks_send_raw+66>
0x00007ffff7bd4243 <+131>: jmp 0x7ffff7bd4219 <iks_send_raw+89>
0x00007ffff7bd4245: data32 nopw %cs:0x0(%rax,%rax,1)
556 return IKS_OK;
0x00007ffff7bd4219 <+89>: xor %eax,%eax
557 }
0x00007ffff7bd421b <+91>: pop %rbx
0x00007ffff7bd421c <+92>: pop %rbp
0x00007ffff7bd421d <+93>: pop %r12
0x00007ffff7bd421f <+95>: retq
End of assembler dump.
More information about the Pkg-voip-maintainers
mailing list