-
Incident report
-
Resolution: Fixed
-
Blocker
-
1.8.7
-
None
-
CentOS 5.6 i686 on Pentium 4 630, 3GB RAM, using dedicated SQL server (PostgreSQL 8.4.8 for production, PostgreSQL 9.0.4 in testing).
I've tried updating to Zabbix 1.8.7 yesterday, but I can't keep the server up. The stack trace is always the same:
8477:20110902:084728.689 server #0 started [main process]
8523:20110902:084805.578 Got signal [signal:11(SIGSEGV),reason:1,refaddr:(nil)]. Crashing ...
8523:20110902:084805.579 ====== Fatal information: ======
8523:20110902:084805.579 Program counter: 0x981453
8523:20110902:084805.579 === Registers: ===
8523:20110902:084805.579 gs = 33 = 51 = 51
8523:20110902:084805.579 fs = 0 = 0 = 0
8523:20110902:084805.579 es = c040007b = 3225419899 = -1069547397
8523:20110902:084805.579 ds = c040007b = 3225419899 = -1069547397
8523:20110902:084805.579 edi = 0 = 0 = 0
8523:20110902:084805.579 esi = 0 = 0 = 0
8523:20110902:084805.579 ebp = bfacd998 = 3215776152 = -1079191144
8523:20110902:084805.579 esp = bfacd97c = 3215776124 = -1079191172
8523:20110902:084805.579 ebx = a65ff4 = 10903540 = 10903540
8523:20110902:084805.579 edx = 1 = 1 = 1
8523:20110902:084805.579 ecx = 0 = 0 = 0
8523:20110902:084805.579 eax = 0 = 0 = 0
8523:20110902:084805.579 trapno = e = 14 = 14
8523:20110902:084805.579 err = 4 = 4 = 4
8523:20110902:084805.579 eip = 981453 = 9966675 = 9966675
8523:20110902:084805.579 cs = 73 = 115 = 115
8523:20110902:084805.579 efl = 10246 = 66118 = 66118
8523:20110902:084805.579 uesp = bfacd97c = 3215776124 = -1079191172
8523:20110902:084805.579 ss = 7b = 123 = 123
8523:20110902:084805.579 === Stack frame: ===
8523:20110902:084805.579 +0x40(%ebp) = ebp + 64 = 00000000 = 0 = 0
8523:20110902:084805.579 +0x3c(%ebp) = ebp + 60 = 00000000 = 0 = 0
8523:20110902:084805.579 +0x38(%ebp) = ebp + 56 = 00007c8a = 31882 = 31882
8523:20110902:084805.579 +0x34(%ebp) = ebp + 52 = 00000000 = 0 = 0
8523:20110902:084805.579 +0x30(%ebp) = ebp + 48 = 00000000 = 0 = 0
8523:20110902:084805.579 +0x2c(%ebp) = ebp + 44 = 0000081f = 2079 = 2079
8523:20110902:084805.580 +0x28(%ebp) = ebp + 40 = 080e4799 = 135153561 = 135153561
8523:20110902:084805.580 +0x24(%ebp) = ebp + 36 = 080c2cf2 = 135015666 = 135015666
8523:20110902:084805.580 +0x20(%ebp) = ebp + 32 = bfacdc28 = 3215776808 = -1079190488
8523:20110902:084805.580 +0x1c(%ebp) = ebp + 28 = 00000001 = 1 = 1
8523:20110902:084805.580 +0x18(%ebp) = ebp + 24 = 00000000 = 0 = 0
8523:20110902:084805.580 +0x14(%ebp) = ebp + 20 = 09db2db8 = 165359032 = 165359032
8523:20110902:084805.580 +0x10(%ebp) = ebp + 16 = 00000004 = 4 = 4
8523:20110902:084805.580 +0x0c(%ebp) = ebp + 12 = 0000001c = 28 = 28
8523:20110902:084805.580 +0x08(%ebp) = ebp + 8 = 00000000 = 0 = 0 <--- call arguments
8523:20110902:084805.580 +0x04(%ebp) = ebp + 4 = 080a18c1 <--- return address
8523:20110902:084805.580 (%ebp) = ebp = bfacd9b8 <--- saved ebp value
8523:20110902:084805.580 0x04(%ebp) = ebp - 4 = 09db6840 = 165374016 = 165374016 <-- local variables
8523:20110902:084805.580 -0x08(%ebp) = ebp - 8 = 00000000 = 0 = 0
8523:20110902:084805.580 -0x0c(%ebp) = ebp - 12 = 00000000 = 0 = 0
8523:20110902:084805.580 -0x10(%ebp) = ebp - 16 = bfacd9b8 = 3215776184 = -1079191112
8523:20110902:084805.580 -0x14(%ebp) = ebp - 20 = 00000000 = 0 = 0
8523:20110902:084805.580 -0x18(%ebp) = ebp - 24 = 00000000 = 0 = 0
8523:20110902:084805.580 -0x1c(%ebp) = ebp - 28 = 00981195 = 9965973 = 9965973
8523:20110902:084805.580 -0x20(%ebp) = ebp - 32 = bfacd988 = 3215776136 = -1079191160
8523:20110902:084805.580 -0x24(%ebp) = ebp - 36 = 0980cd00 = 159436032 = 159436032
8523:20110902:084805.580 -0x28(%ebp) = ebp - 40 = 0000adb8 = 44472 = 44472
8523:20110902:084805.580 -0x2c(%ebp) = ebp - 44 = 00000000 = 0 = 0
8523:20110902:084805.580 -0x30(%ebp) = ebp - 48 = 00000000 = 0 = 0
8523:20110902:084805.580 -0x34(%ebp) = ebp - 52 = 00000000 = 0 = 0
8523:20110902:084805.580 -0x38(%ebp) = ebp - 56 = 00000000 = 0 = 0
8523:20110902:084805.580 -0x3c(%ebp) = ebp - 60 = 00000000 = 0 = 0
8523:20110902:084805.580 -0x40(%ebp) = ebp - 64 = 00000000 = 0 = 0
8523:20110902:084805.580 === Backtrace: ===
8523:20110902:084805.581 11: /home/zabbix/sbin/zabbix_server(print_fatal_info+0xa55) [0x809bb95]
8523:20110902:084805.581 10: /home/zabbix/sbin/zabbix_server [0x809a9d4]
8523:20110902:084805.581 9: [0xa7e440]
8523:20110902:084805.581 8: /home/zabbix/sbin/zabbix_server(zbx_strdup2+0x21) [0x80a18c1]
8523:20110902:084805.581 7: /home/zabbix/sbin/zabbix_server(DBget_history+0x2d2) [0x80c2cf2]
8523:20110902:084805.581 6: /home/zabbix/sbin/zabbix_server(get_value_aggregate+0x8da) [0x8064d6a]
8523:20110902:084805.581 5: /home/zabbix/sbin/zabbix_server(main_poller_loop+0x1642) [0x8062212]
8523:20110902:084805.581 4: /home/zabbix/sbin/zabbix_server(MAIN_ZABBIX_ENTRY+0x6c9) [0x8057949]
8523:20110902:084805.581 3: /home/zabbix/sbin/zabbix_server(daemon_start+0x2b2) [0x809a632]
8523:20110902:084805.581 2: /home/zabbix/sbin/zabbix_server(main+0x222) [0x80570d2]
8523:20110902:084805.581 1: /lib/libc.so.6(__libc_start_main+0xdc) [0x926e9c]
8523:20110902:084805.581 0: /home/zabbix/sbin/zabbix_server [0x80529a1]
8523:20110902:084805.581 === Memory map: ===
8523:20110902:084805.582 00110000-0011c000 r-xp 00000000 fd:00 40901817 /usr/lib/libiksemel.so.3.1.1
8523:20110902:084805.582 0011c000-0011d000 rwxp 0000b000 fd:00 40901817 /usr/lib/libiksemel.so.3.1.1
8523:20110902:084805.582 0011d000-00144000 r-xp 00000000 fd:00 56983560 /lib/libm-2.5.so
8523:20110902:084805.582 00144000-00145000 r-xp 00026000 fd:00 56983560 /lib/libm-2.5.so
8523:20110902:084805.582 00145000-00146000 rwxp 00027000 fd:00 56983560 /lib/libm-2.5.so
8523:20110902:084805.582 0014c000-001e4000 r-xp 00000000 fd:00 40895695 /usr/lib/libnetsnmp.so.10.0.3
8523:20110902:084805.582 001e4000-001e6000 rwxp 00098000 fd:00 40895695 /usr/lib/libnetsnmp.so.10.0.3
8523:20110902:084805.582 001e6000-00205000 rwxp 001e6000 00:00 0
8523:20110902:084805.582 00205000-00212000 r-xp 00000000 fd:00 40901375 /usr/lib/liblber-2.3.so.0.2.31
8523:20110902:084805.582 00212000-00213000 rwxp 0000c000 fd:00 40901375 /usr/lib/liblber-2.3.so.0.2.31
8523:20110902:084805.582 00213000-00228000 r-xp 00000000 fd:00 56983789 /lib/libnsl-2.5.so
8523:20110902:084805.582 00228000-00229000 r-xp 00014000 fd:00 56983789 /lib/libnsl-2.5.so
8523:20110902:084805.582 00229000-0022a000 rwxp 00015000 fd:00 56983789 /lib/libnsl-2.5.so
8523:20110902:084805.582 0022a000-0022c000 rwxp 0022a000 00:00 0
8523:20110902:084805.582 00235000-00262000 r-xp 00000000 fd:00 40901048 /usr/lib/libgssapi_krb5.so.2.2
8523:20110902:084805.582 00262000-00263000 rwxp 0002d000 fd:00 40901048 /usr/lib/libgssapi_krb5.so.2.2
8523:20110902:084805.582 00265000-0026d000 r-xp 00000000 fd:00 40901035 /usr/lib/libkrb5support.so.0.1
8523:20110902:084805.582 0026d000-0026e000 rwxp 00007000 fd:00 40901035 /usr/lib/libkrb5support.so.0.1
8523:20110902:084805.582 00270000-00296000 r-xp 00000000 fd:00 40901043 /usr/lib/libk5crypto.so.3.1
8523:20110902:084805.582 00296000-00297000 rwxp 00025000 fd:00 40901043 /usr/lib/libk5crypto.so.3.1
8523:20110902:084805.582 00299000-0032d000 r-xp 00000000 fd:00 40901045 /usr/lib/libkrb5.so.3.3
8523:20110902:084805.582 0032d000-00330000 rwxp 00093000 fd:00 40901045 /usr/lib/libkrb5.so.3.3
8523:20110902:084805.582 00330000-00369000 r-xp 00000000 fd:00 40896403 /usr/lib/libldap-2.3.so.0.2.31
8523:20110902:084805.583 00369000-0036a000 rwxp 00039000 fd:00 40896403 /usr/lib/libldap-2.3.so.0.2.31
8523:20110902:084805.583 00371000-003b5000 r-xp 00000000 fd:00 56984456 /lib/libssl.so.0.9.8e
8523:20110902:084805.583 003b5000-003b9000 rwxp 00043000 fd:00 56984456 /lib/libssl.so.0.9.8e
8523:20110902:084805.583 003b9000-00495000 r-xp 00000000 fd:00 40901781 /usr/lib/libOpenIPMI.so.0.0.5
8523:20110902:084805.583 00495000-004a4000 rwxp 000db000 fd:00 40901781 /usr/lib/libOpenIPMI.so.0.0.5
8523:20110902:084805.583 004a4000-004a6000 rwxp 004a4000 00:00 0
8523:20110902:084805.583 004a6000-004e4000 r-xp 00000000 fd:00 40901564 /usr/lib/libldap_r-2.3.so.0.2.31
8523:20110902:084805.583 004e4000-004e6000 rwxp 0003d000 fd:00 40901564 /usr/lib/libldap_r-2.3.so.0.2.31
8523:20110902:084805.583 004e6000-004eb000 rwxp 004e6000 00:00 0
8523:20110902:084805.583 004eb000-00564000 r-xp 00000000 fd:00 40896504 /usr/lib/libgnutls.so.13.0.6
8523:20110902:084805.583 00564000-0056a000 rwxp 00078000 fd:00 40896504 /usr/lib/libgnutls.so.13.0.6
8523:20110902:084805.583 0056a000-00580000 r-xp 00000000 fd:00 56984422 /lib/libselinux.so.1
8523:20110902:084805.583 00580000-00582000 rwxp 00015000 fd:00 56984422 /lib/libselinux.so.1
8523:20110902:084805.583 00582000-0058c000 r-xp 00000000 fd:00 56984417 /lib/libnss_files-2.5.so
8523:20110902:084805.583 0058c000-0058d000 r-xp 00009000 fd:00 56984417 /lib/libnss_files-2.5.so
8523:20110902:084805.583 0058d000-0058e000 rwxp 0000a000 fd:00 56984417 /lib/libnss_files-2.5.so
8523:20110902:084805.583 0059f000-005a6000 r-xp 00000000 fd:00 56983629 /lib/libwrap.so.0.7.6
8523:20110902:084805.583 005a6000-005a7000 rwxp 00007000 fd:00 56983629 /lib/libwrap.so.0.7.6
8523:20110902:084805.583 005a7000-005e7000 r-xp 00000000 fd:00 40895930 /usr/lib/libncurses.so.5.5
8523:20110902:084805.583 005e7000-005ef000 rwxp 00040000 fd:00 40895930 /usr/lib/libncurses.so.5.5
8523:20110902:084805.583 005ef000-005f0000 rwxp 005ef000 00:00 0
8523:20110902:084805.583 005f0000-0066e000 r-xp 00000000 fd:00 40901779 /usr/lib/libgcrypt.so.11.5.2
8523:20110902:084805.583 0066e000-00671000 rwxp 0007d000 fd:00 40901779 /usr/lib/libgcrypt.so.11.5.2
8523:20110902:084805.583 0073a000-0073e000 r-xp 00000000 fd:00 56983847 /lib/libnss_dns-2.5.so
8523:20110902:084805.583 0073e000-0073f000 r-xp 00003000 fd:00 56983847 /lib/libnss_dns-2.5.so
8523:20110902:084805.583 0073f000-00740000 rwxp 00004000 fd:00 56983847 /lib/libnss_dns-2.5.so
8523:20110902:084805.583 00796000-007ba000 r-xp 00000000 fd:00 43089965 /usr/pgsql-9.0/lib/libpq.so.5.3
8523:20110902:084805.583 007ba000-007bc000 rwxp 00023000 fd:00 43089965 /usr/pgsql-9.0/lib/libpq.so.5.3
8523:20110902:084805.583 00844000-00866000 r-xp 00000000 fd:00 40895922 /usr/lib/libssh2.so.1.0.1
8523:20110902:084805.583 00866000-00867000 rwxp 00022000 fd:00 40895922 /usr/lib/libssh2.so.1.0.1
8523:20110902:084805.584 008f2000-0090d000 r-xp 00000000 fd:00 56983554 /lib/ld-2.5.so
8523:20110902:084805.584 0090d000-0090e000 r-xp 0001a000 fd:00 56983554 /lib/ld-2.5.so
8523:20110902:084805.584 0090e000-0090f000 rwxp 0001b000 fd:00 56983554 /lib/ld-2.5.so
8523:20110902:084805.584 00911000-00a64000 r-xp 00000000 fd:00 56983557 /lib/libc-2.5.so
8523:20110902:084805.584 00a64000-00a66000 r-xp 00153000 fd:00 56983557 /lib/libc-2.5.so
8523:20110902:084805.584 00a66000-00a67000 rwxp 00155000 fd:00 56983557 /lib/libc-2.5.so
8523:20110902:084805.584 00a67000-00a6a000 rwxp 00a67000 00:00 0
8523:20110902:084805.584 00a6c000-00a6f000 r-xp 00000000 fd:00 56983595 /lib/libdl-2.5.so
8523:20110902:084805.584 00a6f000-00a70000 r-xp 00002000 fd:00 56983595 /lib/libdl-2.5.so
8523:20110902:084805.584 00a70000-00a71000 rwxp 00003000 fd:00 56983595 /lib/libdl-2.5.so
8523:20110902:084805.584 00a7e000-00a7f000 r-xp 00a7e000 00:00 0 [vdso]
8523:20110902:084805.584 00a9e000-00ab3000 r-xp 00000000 fd:00 56983627 /lib/libpthread-2.5.so
8523:20110902:084805.584 00ab3000-00ab4000 r-xp 00015000 fd:00 56983627 /lib/libpthread-2.5.so
8523:20110902:084805.584 00ab4000-00ab5000 rwxp 00016000 fd:00 56983627 /lib/libpthread-2.5.so
8523:20110902:084805.584 00ab5000-00ab7000 rwxp 00ab5000 00:00 0
8523:20110902:084805.584 00ab9000-00acb000 r-xp 00000000 fd:00 40901374 /usr/lib/libz.so.1.2.3
8523:20110902:084805.584 00acb000-00acc000 rwxp 00011000 fd:00 40901374 /usr/lib/libz.so.1.2.3
8523:20110902:084805.584 00ace000-00ad4000 r-xp 00000000 fd:00 40901782 /usr/lib/libgdbm.so.2.0.0
8523:20110902:084805.584 00ad4000-00ad5000 rwxp 00005000 fd:00 40901782 /usr/lib/libgdbm.so.2.0.0
8523:20110902:084805.584 00ad7000-00adc000 r-xp 00000000 fd:00 40901783 /usr/lib/libOpenIPMIposix.so.0.0.1
8523:20110902:084805.584 00adc000-00add000 rwxp 00004000 fd:00 40901783 /usr/lib/libOpenIPMIposix.so.0.0.1
8523:20110902:084805.584 00adf000-00ae6000 r-xp 00000000 fd:00 40896395 /usr/lib/libOpenIPMIutils.so.0.0.1
8523:20110902:084805.584 00ae6000-00ae7000 rwxp 00006000 fd:00 40896395 /usr/lib/libOpenIPMIutils.so.0.0.1
8523:20110902:084805.584 00ae8000-00b23000 r-xp 00000000 fd:00 56984414 /lib/libsepol.so.1
8523:20110902:084805.584 00b23000-00b24000 rwxp 0003b000 fd:00 56984414 /lib/libsepol.so.1
8523:20110902:084805.585 00b24000-00b2e000 rwxp 00b24000 00:00 0
8523:20110902:084805.585 00b3b000-00c6a000 r-xp 00000000 fd:00 56984455 /lib/libcrypto.so.0.9.8e
8523:20110902:084805.585 00c6a000-00c7d000 rwxp 0012e000 fd:00 56984455 /lib/libcrypto.so.0.9.8e
8523:20110902:084805.585 00c7d000-00c81000 rwxp 00c7d000 00:00 0
8523:20110902:084805.585 00c83000-00c8c000 r-xp 00000000 fd:00 56984457 /lib/libcrypt-2.5.so
8523:20110902:084805.585 00c8c000-00c8d000 r-xp 00008000 fd:00 56984457 /lib/libcrypt-2.5.so
8523:20110902:084805.585 00c8d000-00c8e000 rwxp 00009000 fd:00 56984457 /lib/libcrypt-2.5.so
8523:20110902:084805.585 00c8e000-00cb5000 rwxp 00c8e000 00:00 0
8523:20110902:084805.585 00cb7000-00cc2000 r-xp 00000000 fd:00 56983583 /lib/libgcc_s-4.1.2-20080825.so.1
8523:20110902:084805.585 00cc2000-00cc3000 rwxp 0000a000 fd:00 56983583 /lib/libgcc_s-4.1.2-20080825.so.1
8523:20110902:084805.585 00cc5000-00cdd000 r-xp 00000000 fd:00 40901453 /usr/lib/libsasl2.so.2.0.22
8523:20110902:084805.585 00cdd000-00cde000 rwxp 00017000 fd:00 40901453 /usr/lib/libsasl2.so.2.0.22
8523:20110902:084805.585 00ce0000-00ce3000 r-xp 00000000 fd:00 40896502 /usr/lib/libgpg-error.so.0.3.0
8523:20110902:084805.585 00ce3000-00ce4000 rwxp 00002000 fd:00 40896502 /usr/lib/libgpg-error.so.0.3.0
8523:20110902:084805.585 00ce7000-00d17000 r-xp 00000000 fd:00 40896788 /usr/lib/libidn.so.11.5.19
8523:20110902:084805.585 00d17000-00d18000 rwxp 0002f000 fd:00 40896788 /usr/lib/libidn.so.11.5.19
8523:20110902:084805.585 00d57000-00d67000 r-xp 00000000 fd:00 56983787 /lib/libresolv-2.5.so
8523:20110902:084805.585 00d67000-00d68000 r-xp 0000f000 fd:00 56983787 /lib/libresolv-2.5.so
8523:20110902:084805.585 00d68000-00d69000 rwxp 00010000 fd:00 56983787 /lib/libresolv-2.5.so
8523:20110902:084805.585 00d69000-00d6b000 rwxp 00d69000 00:00 0
8523:20110902:084805.585 00d6d000-00d6f000 r-xp 00000000 fd:00 56983593 /lib/libkeyutils-1.2.so
8523:20110902:084805.585 00d6f000-00d70000 rwxp 00001000 fd:00 56983593 /lib/libkeyutils-1.2.so
8523:20110902:084805.585 00d72000-00dad000 r-xp 00000000 fd:00 40896762 /usr/lib/libcurl.so.3.0.0
8523:20110902:084805.585 00dad000-00dae000 rwxp 0003b000 fd:00 40896762 /usr/lib/libcurl.so.3.0.0
8523:20110902:084805.585 00db3000-00db5000 r-xp 00000000 fd:00 56984454 /lib/libcom_err.so.2.1
8523:20110902:084805.585 00db5000-00db6000 rwxp 00001000 fd:00 56984454 /lib/libcom_err.so.2.1
8523:20110902:084805.585 08048000-080e9000 r-xp 00000000 fd:00 35815456 /home/zabbix/sbin/zabbix_server.1.8.7
8523:20110902:084805.585 080e9000-08102000 rw-p 000a0000 fd:00 35815456 /home/zabbix/sbin/zabbix_server.1.8.7
8523:20110902:084805.585 08102000-08108000 rw-p 08102000 00:00 0
8523:20110902:084805.586 09d43000-09de6000 rw-p 09d43000 00:00 0 [heap]
8523:20110902:084805.586 aed04000-aed0b000 r--s 00000000 fd:00 40992775 /usr/lib/gconv/gconv-modules.cache
8523:20110902:084805.586 aed0b000-aef0b000 r--p 00000000 fd:00 42729474 /usr/lib/locale/locale-archive
8523:20110902:084805.586 aef0b000-aef15000 rw-s 00000000 00:09 5079052 /SYSV5300802e (deleted)
8523:20110902:084805.586 aef15000-af3e2000 rw-s 00000000 00:09 5046283 /SYSV7300802e (deleted)
8523:20110902:084805.586 af3e2000-b0f16000 rw-s 00000000 00:09 5013514 /SYSV6700802e (deleted)
8523:20110902:084805.586 b0f16000-b1f17000 rw-s 00000000 00:09 4980745 /SYSV7400802e (deleted)
8523:20110902:084805.586 b1f17000-b5f18000 rw-s 00000000 00:09 4947976 /SYSV7800802e (deleted)
8523:20110902:084805.586 b5f18000-b7f20000 rw-s 00000000 00:09 4915206 /SYSV6800802e (deleted)
8523:20110902:084805.586 b7f20000-b7f2a000 rw-p b7f20000 00:00 0
8523:20110902:084805.586 b7f30000-b7f32000 rw-p b7f30000 00:00 0
8523:20110902:084805.586 bfabc000-bfb21000 rw-p bff99000 00:00 0 [stack]
8523:20110902:084805.586 ================================
8523:20110902:084805.586 Please consider attaching a disassembly listing to your bug report.
8523:20110902:084805.586 This listing can be produced with, e.g., objdump -DSswx zabbix_server.
8523:20110902:084805.586 ================================
8477:20110902:084805.587 One child process died (PID:8523,exitcode/signal:255). Exiting ...
8477:20110902:084807.712 Syncing history data...
8477:20110902:084809.416 Syncing history data... done.
8477:20110902:084809.417 Syncing trends data...
8477:20110902:084817.027 Syncing trends data... done.
8477:20110902:084817.027 Zabbix Server stopped. Zabbix 1.8.7 (revision 21394).
When reverting to 1.8.6 the problem goes away.
Both 1.8.6 and 1.8.7 (and all earlier versions I used) are compiled manually from sources, with:
CFLAGS="-pipe -O3 -fno-strict-aliasing -funroll-loops" ./configure --prefix=/home/zabbix --with-net-snmp --with-libcurl --with-pgsql --enable-server --enable-agent --with-ldap --with-openipmi --with-jabber --with-ssh2 && make