zabbix agent has hung and is not responding. the only change done on that system was enabling of ntp time sync (which at most would gradually shift time by 6 seconds)
server kept working just fine

last entry in agent logfile :

Can't parse list of active checks. Can't open JSON object ""

strace of agent master process :

waitpid(7286

pid 7286 seems to be the one that's stuck :

open("/proc/stat", O_RDONLY) = 5
fstat64(5,

{st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb786d000
read(5, "cpu 16611 3264 65328 9831464 43"..., 1024) = 854
close(5) = 0
munmap(0xb786d000, 4096) = 0
time(NULL) = 1294585092
open("/proc/stat", O_RDONLY) = 5
fstat64(5, {st_mode=S_IFREG|0444, st_size=0, ...}

) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb786d000
read(5, "cpu 16611 3264 65328 9831464 43"..., 1024) = 854
close(5) = 0
munmap(0xb786d000, 4096) = 0
time(NULL) = 1294585092
open("/proc/diskstats", O_RDONLY) = 5
fstat64(5,

{st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb786d000
read(5, " 2 0 fd0 0 0 0 0 0 0 0 0"..., 1024) = 1024
read(5, "0 sr0 24 0 192 78 0 0 0 0 0 78 7"..., 1024) = 233
read(5, "", 1024) = 0
close(5) = 0
munmap(0xb786d000, 4096) = 0
rt_sigprocmask(SIG_BLOCK, [CHLD], [], 8) = 0
rt_sigaction(SIGCHLD, NULL, {SIG_DFL, [], 0}, 8) = 0
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
nanosleep({1, 0}, 0xbff7cb24) = 0
time(NULL) = 1294585093
open("/proc/stat", O_RDONLY) = 5
fstat64(5, {st_mode=S_IFREG|0444, st_size=0, ...}

) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb786d000
read(5, "cpu 16611 3264 65329 9831557 43"..., 1024) = 854
close(5) = 0
munmap(0xb786d000, 4096) = 0
time(NULL) = 1294585093
open("/proc/stat", O_RDONLY) = 5
fstat64(5,

{st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb786d000
read(5, "cpu 16611 3264 65329 9831557 43"..., 1024) = 854
close(5) = 0
munmap(0xb786d000, 4096) = 0
time(NULL) = 1294585093
open("/proc/diskstats", O_RDONLY) = 5
fstat64(5, {st_mode=S_IFREG|0444, st_size=0, ...}

) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb786d000
read(5, " 2 0 fd0 0 0 0 0 0 0 0 0"..., 1024) = 1024
read(5, "0 sr0 24 0 192 78 0 0 0 0 0 78 7"..., 1024) = 233
read(5, "", 1024) = 0
close(5) = 0
munmap(0xb786d000, 4096) = 0
rt_sigprocmask(SIG_BLOCK, [CHLD], [], 8) = 0
rt_sigaction(SIGCHLD, NULL,

{SIG_DFL, [], 0}, 8) = 0
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
nanosleep({1, 0}, 0xbff7cb24) = 0
time(NULL) = 1294585094
open("/proc/stat", O_RDONLY) = 5
fstat64(5, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb786d000
read(5, "cpu 16611 3264 65329 9831651 43"..., 1024) = 854
close(5) = 0
munmap(0xb786d000, 4096) = 0
time(NULL) = 1294585094
open("/proc/stat", O_RDONLY) = 5
fstat64(5, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb786d000
read(5, "cpu 16611 3264 65330 9831651 43"..., 1024) = 854
close(5) = 0
munmap(0xb786d000, 4096) = 0
time(NULL) = 1294585094
open("/proc/diskstats", O_RDONLY) = 5
fstat64(5, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb786d000
read(5, " 2 0 fd0 0 0 0 0 0 0 0 0"..., 1024) = 1024
read(5, "0 sr0 24 0 192 78 0 0 0 0 0 78 7"..., 1024) = 233
read(5, "", 1024) = 0
close(5) = 0
munmap(0xb786d000, 4096) = 0
rt_sigprocmask(SIG_BLOCK, [CHLD], [], 8) = 0
rt_sigaction(SIGCHLD, NULL, {SIG_DFL, [], 0}

, 8) = 0
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
nanosleep(

{1, 0}, 0xbff7cb24) = 0
time(NULL) = 1294585095
open("/proc/stat", O_RDONLY) = 5
fstat64(5, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb786d000
read(5, "cpu 16611 3264 65330 9831742 43"..., 1024) = 854
close(5) = 0
munmap(0xb786d000, 4096) = 0
time(NULL) = 1294585095
open("/proc/stat", O_RDONLY) = 5
fstat64(5, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb786d000
read(5, "cpu 16611 3264 65330 9831742 43"..., 1024) = 854
close(5) = 0
munmap(0xb786d000, 4096) = 0
time(NULL) = 1294585095
open("/proc/diskstats", O_RDONLY) = 5
fstat64(5, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb786d000
read(5, " 2 0 fd0 0 0 0 0 0 0 0 0"..., 1024) = 1024
read(5, "0 sr0 24 0 192 78 0 0 0 0 0 78 7"..., 1024) = 233
read(5, "", 1024) = 0
close(5) = 0
munmap(0xb786d000, 4096) = 0
rt_sigprocmask(SIG_BLOCK, [CHLD], [], 8) = 0
rt_sigaction(SIGCHLD, NULL, {SIG_DFL, [], 0}, 8) = 0
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
nanosleep({1, 0}

, 0xbff7cb24) = 0
time(NULL) = 1294585096
open("/proc/stat", O_RDONLY) = 5
fstat64(5,

{st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb786d000
read(5, "cpu 16611 3264 65331 9831842 43"..., 1024) = 854
close(5) = 0
munmap(0xb786d000, 4096) = 0
time(NULL) = 1294585096
open("/proc/stat", O_RDONLY) = 5
fstat64(5, {st_mode=S_IFREG|0444, st_size=0, ...}

) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb786d000
read(5, "cpu 16611 3264 65331 9831842 43"..., 1024) = 854
close(5) = 0
munmap(0xb786d000, 4096) = 0
time(NULL) = 1294585096
open("/proc/diskstats", O_RDONLY) = 5
fstat64(5,

{st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb786d000
read(5, " 2 0 fd0 0 0 0 0 0 0 0 0"..., 1024) = 1024
read(5, "0 sr0 24 0 192 78 0 0 0 0 0 78 7"..., 1024) = 233
read(5, "", 1024) = 0
close(5) = 0
munmap(0xb786d000, 4096) = 0
rt_sigprocmask(SIG_BLOCK, [CHLD], [], 8) = 0
rt_sigaction(SIGCHLD, NULL, {SIG_DFL, [], 0}, 8) = 0
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
nanosleep({1, 0}, 0xbff7cb24) = 0
time(NULL) = 1294585097
open("/proc/stat", O_RDONLY) = 5
fstat64(5, {st_mode=S_IFREG|0444, st_size=0, ...}

) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb786d000
read(5, "cpu 16611 3264 65331 9831942 43"..., 1024) = 854
close(5) = 0
munmap(0xb786d000, 4096) = 0
time(NULL) = 1294585097
open("/proc/stat", O_RDONLY) = 5
fstat64(5,

{st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb786d000
read(5, "cpu 16611 3264 65331 9831942 43"..., 1024) = 854
close(5) = 0
munmap(0xb786d000, 4096) = 0
time(NULL) = 1294585097
open("/proc/diskstats", O_RDONLY) = 5
fstat64(5, {st_mode=S_IFREG|0444, st_size=0, ...}

) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb786d000
read(5, " 2 0 fd0 0 0 0 0 0 0 0 0"..., 1024) = 1024
read(5, "0 sr0 24 0 192 78 0 0 0 0 0 78 7"..., 1024) = 233
read(5, "", 1024) = 0
close(5) = 0
munmap(0xb786d000, 4096) = 0
rt_sigprocmask(SIG_BLOCK, [CHLD], [], 8) = 0
rt_sigaction(SIGCHLD, NULL,

{SIG_DFL, [], 0}, 8) = 0
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
nanosleep({1, 0}, 0xbff7cb24) = 0
time(NULL) = 1294585098
open("/proc/stat", O_RDONLY) = 5
fstat64(5, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb786d000
read(5, "cpu 16611 3264 65332 9832042 43"..., 1024) = 854
close(5) = 0
munmap(0xb786d000, 4096) = 0
time(NULL) = 1294585098
open("/proc/stat", O_RDONLY) = 5
fstat64(5, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb786d000
read(5, "cpu 16611 3264 65332 9832042 43"..., 1024) = 854
close(5) = 0
munmap(0xb786d000, 4096) = 0
time(NULL) = 1294585098
open("/proc/diskstats", O_RDONLY) = 5
fstat64(5, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb786d000
read(5, " 2 0 fd0 0 0 0 0 0 0 0 0"..., 1024) = 1024
read(5, "0 sr0 24 0 192 78 0 0 0 0 0 78 7"..., 1024) = 233
read(5, "", 1024) = 0
close(5) = 0
munmap(0xb786d000, 4096) = 0
rt_sigprocmask(SIG_BLOCK, [CHLD], [], 8) = 0
rt_sigaction(SIGCHLD, NULL, {SIG_DFL, [], 0}

, 8) = 0
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
nanosleep(

{1, 0}, 0xbff7cb24) = 0
time(NULL) = 1294585099
open("/proc/stat", O_RDONLY) = 5
fstat64(5, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb786d000
read(5, "cpu 16611 3264 65332 9832141 43"..., 1024) = 854
close(5) = 0
munmap(0xb786d000, 4096) = 0
time(NULL) = 1294585099
open("/proc/stat", O_RDONLY) = 5
fstat64(5, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb786d000
read(5, "cpu 16611 3264 65332 9832141 43"..., 1024) = 854
close(5) = 0
munmap(0xb786d000, 4096) = 0
time(NULL) = 1294585099
open("/proc/diskstats", O_RDONLY) = 5
fstat64(5, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb786d000
read(5, " 2 0 fd0 0 0 0 0 0 0 0 0"..., 1024) = 1024
read(5, "0 sr0 24 0 192 78 0 0 0 0 0 78 7"..., 1024) = 233
read(5, "", 1024) = 0
close(5) = 0
munmap(0xb786d000, 4096) = 0
rt_sigprocmask(SIG_BLOCK, [CHLD], [], 8) = 0
rt_sigaction(SIGCHLD, NULL, {SIG_DFL, [], 0}, 8) = 0
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
nanosleep({1, 0}

, 0xbff7cb24) = 0
time(NULL) = 1294585100
open("/proc/stat", O_RDONLY) = 5
fstat64(5,

{st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb786d000
read(5, "cpu 16611 3264 65333 9832241 43"..., 1024) = 854
close(5) = 0
munmap(0xb786d000, 4096) = 0
time(NULL) = 1294585100
open("/proc/stat", O_RDONLY) = 5
fstat64(5, {st_mode=S_IFREG|0444, st_size=0, ...}

) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb786d000
read(5, "cpu 16611 3264 65333 9832241 43"..., 1024) = 854
close(5) = 0
munmap(0xb786d000, 4096) = 0
time(NULL) = 1294585100
open("/proc/diskstats", O_RDONLY) = 5
fstat64(5,

{st_mode=S_IFREG|0444, st_size=0, ...}

) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb786d000
read(5, " 2 0 fd0 0 0 0 0 0 0 0 0"..., 1024) = 1024
read(5, "0 sr0 24 0 192 78 0 0 0 0 0 78 7"..., 1024) = 233
read(5, "", 1024) = 0
close(5) = 0
munmap(0xb786d000, 4096) = 0