core file in user directory, what program failed?

susestylin · December 6, 2011, 5:37pm

Hi everyone,

I’ve got a problem with LXDE failing to launch through a FreeNX session. NX authentication passes, but after that the session fails to continue. One log file shows messages like below, and a core file is created in the user’s home directory. How can I analyze the core file to know what the problem is?

*** glibc detected *** /usr/lib/NX/nxagent: free(): invalid pointer: 0x084f0e48

…
xrdb: Connection refused
xrdb: Can’t open display ‘:1002’

susestylin · December 6, 2011, 5:41pm

Here’s the full stacktrace:

*** glibc detected *** /usr/lib/NX/nxagent: free(): invalid pointer: 0x084f0e48

======= Backtrace: =========
/lib/libc.so.6(+0x6ed8b)[0xb73a0d8b]
/lib/libc.so.6(cfree+0x9b)[0xb73a528b]
/usr/lib/NX/nxagent[0x80b4830]
/usr/lib/NX/nxagent[0x80b5c5e]
/usr/lib/NX/nxagent[0x80c2e9d]
/usr/lib/NX/nxagent[0x80543c9]
/lib/libc.so.6(__libc_start_main+0xf3)[0xb734b003]
/usr/lib/NX/nxagent[0x8054cd9]
======= Memory map: ========
08048000-08488000 r-xp 00000000 08:02 2444 /usr/lib/NX/nxagent
08488000-08489000 r–p 00440000 08:02 2444 /usr/lib/NX/nxagent
08489000-084ae000 rw-p 00441000 08:02 2444 /usr/lib/NX/nxagent
084ae000-08775000 rw-p 00000000 00:00 0 [heap]
b7127000-b718d000 rw-p 00000000 00:00 0
b718d000-b718f000 r-xp 00000000 08:02 2280 /usr/lib/libXdamage.so.1.1.0
b718f000-b7190000 r–p 00001000 08:02 2280 /usr/lib/libXdamage.so.1.1.0
b7190000-b7191000 rw-p 00002000 08:02 2280 /usr/lib/libXdamage.so.1.1.0
b7191000-b7198000 r-xp 00000000 08:02 2381 /usr/lib/libXrandr.so.2.2.0
b7198000-b7199000 r–p 00006000 08:02 2381 /usr/lib/libXrandr.so.2.2.0
b7199000-b719a000 rw-p 00007000 08:02 2381 /usr/lib/libXrandr.so.2.2.0
b719a000-b719b000 rw-p 00000000 00:00 0
b719b000-b71a0000 r-xp 00000000 08:02 2283 /usr/lib/libXtst.so.6.1.0
b71a0000-b71a1000 r–p 00004000 08:02 2283 /usr/lib/libXtst.so.6.1.0
b71a1000-b71a2000 rw-p 00005000 08:02 2283 /usr/lib/libXtst.so.6.1.0
b71a2000-b71be000 r-xp 00000000 08:02 5981 /lib/libgcc_s.so.1
b71be000-b71bf000 r–p 0001b000 08:02 5981 /lib/libgcc_s.so.1
b71bf000-b71c0000 rw-p 0001c000 08:02 5981 /lib/libgcc_s.so.1
b71c0000-b72a2000 r-xp 00000000 08:02 23397 /usr/lib/libstdc++.so.6.0.16
b72a2000-b72a6000 r–p 000e2000 08:02 23397 /usr/lib/libstdc++.so.6.0.16
b72a6000-b72a7000 rw-p 000e6000 08:02 23397 /usr/lib/libstdc++.so.6.0.16
b72a7000-b72ae000 rw-p 00000000 00:00 0
b72ae000-b72ee000 r-xp 00000000 08:02 2403 /usr/lib/libjpeg.so.62.0.0
b72ee000-b72ef000 r–p 00040000 08:02 2403 /usr/lib/libjpeg.so.62.0.0
b72ef000-b72f0000 rw-p 00041000 08:02 2403 /usr/lib/libjpeg.so.62.0.0
b72f0000-b7300000 rw-p 00000000 00:00 0
b7300000-b732a000 r-xp 00000000 08:02 187 /usr/lib/libpng12.so.0.46.0
b732a000-b732b000 r–p 00029000 08:02 187 /usr/lib/libpng12.so.0.46.0
b732b000-b732c000 rw-p 0002a000 08:02 187 /usr/lib/libpng12.so.0.46.0
b732c000-b732d000 rw-p 00000000 00:00 0
b732d000-b7330000 r-xp 00000000 08:02 5893 /lib/libdl-2.14.1.so
b7330000-b7331000 r–p 00002000 08:02 5893 /lib/libdl-2.14.1.so
b7331000-b7332000 rw-p 00003000 08:02 5893 /lib/libdl-2.14.1.so
b7332000-b7498000 r-xp 00000000 08:02 76 /lib/libc-2.14.1.so
b7498000-b749a000 r–p 00165000 08:02 76 /lib/libc-2.14.1.so
b749a000-b749b000 rw-p 00167000 08:02 76 /lib/libc-2.14.1.so
b749b000-b749e000 rw-p 00000000 00:00 0
b749e000-b74a0000 r-xp 00000000 08:02 3056 /usr/lib/libXcomposite.so.1.0.0
b74a0000-b74a1000 r–p 00001000 08:02 3056 /usr/lib/libXcomposite.so.1.0.0
b74a1000-b74a2000 rw-p 00002000 08:02 3056 /usr/lib/libXcomposite.so.1.0.0
b74a2000-b74a7000 r-xp 00000000 08:02 3055 /usr/lib/libXfixes.so.3.1.0
b74a7000-b74a8000 r–p 00004000 08:02 3055 /usr/lib/libXfixes.so.3.1.0
b74a8000-b74a9000 rw-p 00005000 08:02 3055 /usr/lib/libXfixes.so.3.1.0
b74a9000-b74b2000 r-xp 00000000 08:02 2404 /usr/lib/NX/lib/libXrender.so.1
.2.2
b74b2000-b74b3000 r–p 00008000 08:02 2404 /usr/lib/NX/lib/libXrender.so.1
.2.2
b74b3000-b74b4000 rw-p 00009000 08:02 2404 /usr/lib/NX/lib/libXrender.so.1
.2.2
b74b4000-b74b5000 rw-p 00000000 00:00 0
b74b5000-b74bf000 r-xp 00000000 08:02 612 /usr/lib/NX/lib/libXcompshad.so
.3.4.0
b74bf000-b74c0000 r–p 00009000 08:02 612 /usr/lib/NX/lib/libXcompshad.so
.3.4.0
b74c0000-b74c1000 rw-p 0000a000 08:02 612 /usr/lib/NX/lib/libXcompshad.so
.3.4.0
b74c1000-b74cf000 r-xp 00000000 08:02 532 /usr/lib/NX/lib/libXcompext.so.
3.4.0
b74cf000-b74d0000 r–p 0000d000 08:02 532 /usr/lib/NX/lib/libXcompext.so.
3.4.0
b74d0000-b74d1000 rw-p 0000e000 08:02 532 /usr/lib/NX/lib/libXcompext.so.
3.4.0
b74d1000-b74d2000 rw-p 00000000 00:00 0
b74d2000-b75c7000 r-xp 00000000 08:02 526 /usr/lib/NX/lib/libXcomp.so.3.4
.0
b75c7000-b75c8000 —p 000f5000 08:02 526 /usr/lib/NX/lib/libXcomp.so.3.4
.0
b75c8000-b75cb000 r–p 000f5000 08:02 526 /usr/lib/NX/lib/libXcomp.so.3.4
.0
b75cb000-b75cc000 rw-p 000f8000 08:02 526 /usr/lib/NX/lib/libXcomp.so.3.4
.0
b75cc000-b75d1000 rw-p 00000000 00:00 0
b75d1000-b75e2000 r-xp 00000000 08:02 3111 /usr/lib/libXpm.so.4.11.0
b75e2000-b75e3000 r–p 00010000 08:02 3111 /usr/lib/libXpm.so.4.11.0
b75e3000-b75e4000 rw-p 00011000 08:02 3111 /usr/lib/libXpm.so.4.11.0
b75e4000-b760d000 r-xp 00000000 08:02 5931 /lib/libm-2.14.1.so
b760d000-b760e000 r–p 00028000 08:02 5931 /lib/libm-2.14.1.so
b760e000-b760f000 rw-p 00029000 08:02 5931 /lib/libm-2.14.1.so
b760f000-b7610000 rw-p 00000000 00:00 0
b7610000-b7626000 r-xp 00000000 08:02 5091 /lib/libz.so.1.2.5
b7626000-b7627000 r–p 00015000 08:02 5091 /lib/libz.so.1.2.5
b7627000-b7628000 rw-p 00016000 08:02 5091 /lib/libz.so.1.2.5
b7628000-b7730000 r-xp 00000000 08:02 242 /usr/lib/NX/lib/libX11.so.6.2
b7730000-b7731000 —p 00108000 08:02 242 /usr/lib/NX/lib/libX11.so.6.2
b7731000-b7732000 r–p 00108000 08:02 242 /usr/lib/NX/lib/libX11.so.6.2
b7732000-b7735000 rw-p 00109000 08:02 242 /usr/lib/NX/lib/libX11.so.6.2
b7735000-b7736000 rw-p 00000000 00:00 0
b7736000-b7746000 r-xp 00000000 08:02 2399 /usr/lib/NX/lib/libXext.so.6.4
b7746000-b7747000 r–p 0000f000 08:02 2399 /usr/lib/NX/lib/libXext.so.6.4
b7747000-b7748000 rw-p 00010000 08:02 2399 /usr/lib/NX/lib/libXext.so.6.4
b7748000-b77cc000 r-xp 00000000 08:02 17213 /usr/lib/libfreetype.so.6.7.2
b77cc000-b77d0000 r–p 00083000 08:02 17213 /usr/lib/libfreetype.so.6.7.2
b77d0000-b77d1000 rw-p 00087000 08:02 17213 /usr/lib/libfreetype.so.6.7.2
b77d6000-b77d7000 rw-p 00000000 00:00 0
b77d7000-b77e0000 r-xp 00000000 08:02 3134 /usr/lib/libXcursor.so.1.0.2
b77e0000-b77e1000 r–p 00008000 08:02 3134 /usr/lib/libXcursor.so.1.0.2
b77e1000-b77e2000 rw-p 00009000 08:02 3134 /usr/lib/libXcursor.so.1.0.2
b77e2000-b77e5000 rw-p 00000000 00:00 0
b77e5000-b7804000 r-xp 00000000 08:02 3402 /lib/ld-2.14.1.so
b7804000-b7805000 r–p 0001f000 08:02 3402 /lib/ld-2.14.1.so
b7805000-b7806000 rw-p 00020000 08:02 3402 /lib/ld-2.14.1.so
bfa9b000-bfabc000 rw-p 00000000 00:00 0 [stack]
ffffe000-fffff000 r-xp 00000000 00:00 0 [vdso]
Warning: Parent process appears to be dead. Exiting keeper.
xrdb: Connection refused
xrdb: Can’t open display ‘:1002’

djh-novell · December 6, 2011, 6:05pm

susestylin wrote:
> How can I analyze the core file to know what the problem is?

Start by typing man core and reading what’s there!

robin_listas · December 6, 2011, 8:28pm

On 2011-12-06 18:05, Dave Howorth wrote:
> Start by typing man core and reading what’s there!

I did, but I could not see a way of learning the name of the program that
produced it.

–
Cheers / Saludos,

Carlos E. R.
(from 11.4 x86_64 “Celadon” at Telcontar)

robin_listas · December 6, 2011, 8:38pm

On 2011-12-06 17:46, susestylin wrote:
> Here’s the full stacktrace:

You should use code tags for this - advanced editor, # button.

>
> *** glibc detected *** /usr/lib/NX/nxagent: free(): invalid pointer:
> 0x084f0e48
> ***

To me, this means that glibc detected an invalid operation and halted the
program. It is a bug in nxagent.

I don’t know if there are options to avoid the halting or not.

–
Cheers / Saludos,

Carlos E. R.
(from 11.4 x86_64 “Celadon” at Telcontar)

ken_yap · December 6, 2011, 10:17pm

file core

should tell you what program produced it, IIRC.

robin_listas · December 7, 2011, 4:13am

On 2011-12-06 22:26, ken yap wrote:
>
> Code:
> --------------------
> file core
> --------------------
>
>
> should tell you what program produced it, IIRC.

Curious.


>
> Telcontar:~ # ls -lh /core
> -rw------- 1 root root 13M Jun  3  2011 /core
> Telcontar:~ # file /core
> /core: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from '/usr/bin/nspluginwrapper -a -u'
> Telcontar:~ #

It seems to be so.

–
Cheers / Saludos,

Carlos E. R.
(from 11.4 x86_64 “Celadon” at Telcontar)

ken_yap · December 7, 2011, 5:09am

There’s another way to get the name of the process generating the core file, and this is also useful in case you are unlucky enough to have more than one crashing program. If you look at man 5 core, you see that you can ahead of time set /proc/sys/kernel/core_pattern to use various info in the filename, and one piece of info is the name of the executable.

susestylin · December 7, 2011, 5:39am

RTFMFTW. man core led me to run gdb on the core file, which gave me a better stacktrace:

#0  0xb734e8c5 in __GI_raise (sig=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:64
#1  0xb73501d5 in __GI_abort () at abort.c:93
#2  0xb73895ca in __libc_message (do_abort=2, fmt=0xb745077c "*** glibc detected *** %s: %s: 0x%s ***
")
    at ../sysdeps/unix/sysv/linux/libc_fatal.c:198
#3  0xb738fd8b in malloc_printerr (action=<optimized out>, str=<optimized out>, ptr=0x84f0e48) at malloc.c:6283
#4  0xb739428b in __GI___libc_free (mem=0x84f0e48) at malloc.c:3699
#5  0x080b4830 in nxagentMakeIcon ()
#6  0x080b5c5e in nxagentOpenDisplay ()
#7  0x080c2e9d in InitOutput ()
#8  0x080543c9 in main ()

It also gave a little more detail for stack frame #2:

#2  0xb73895ca in __libc_message (do_abort=2, fmt=0xb745077c "*** glibc detected *** %s: %s: 0x%s ***
")
    at ../sysdeps/unix/sysv/linux/libc_fatal.c:198
198     ../sysdeps/unix/sysv/linux/libc_fatal.c: No such file or directory.
        in ../sysdeps/unix/sysv/linux/libc_fatal.c

This does suggest that perhaps there is a bug. Searching on nxagentMakeIcon, I found this note on the NX site: http://www.nomachine.com/tr/view.php?id=TR10I02622

This page lists a bug:

**The nxagent process could fail to create an icon**

The nxagent process could crash when creating an icon using the Xpm library.
Debugging shows that nxagent crashes in the function nxagentMakeIcon.

More detail is now required: NX sessions consistently work with users created after this machine was upgraded to 12.1, but sessions for users created on 11.4 consistently crash. How do I figure out what the invalid file or directory is to find out why old users are != new users?

djh-novell · December 7, 2011, 1:27pm

susestylin wrote:
> RTFMFTW. man core led me to run gdb on the core file, which gave me a
> better stacktrace:

Carlos, there’s the answer to your question to me

> More detail is now required: NX sessions consistently work with users
> created after this machine was upgraded to 12.1, but sessions for users
> created on 11.4 consistently crash. How do I figure out what the
> invalid file or directory is to find out why old users are != new users?

Is the info not in the core file somewhere?

Other ways to get it might be to strace the program to see what it
opens. Or use lsof to see what it opens, if it runs for long enough. Or
single-step the program with gdb, of course.

robin_listas · December 8, 2011, 1:13am

On 2011-12-07 13:27, Dave Howorth wrote:
> susestylin wrote:
>> > RTFMFTW. man core led me to run gdb on the core file, which gave me a
>> > better stacktrace:
> Carlos, there’s the answer to your question to me

gdb is a little to complex for me, and I just wanted to know which program
had coredumped. That info is given by “file core”, so that’s enough for me,
for the moment

Just for curiosity, I tried “gdb /core” and it said:

“/core”: not in executable format: File format not recognized

so it is not that simple O:-)

–
Cheers / Saludos,

Carlos E. R.
(from 11.4 x86_64 “Celadon” at Telcontar)

susestylin · December 8, 2011, 2:30am

If you use “gdb -c core”, it will complain but will tell you “Core was generated by ‘/prog/xxx -opts’.”

Once you know what program, then run “gdb -c core /prog/xxx” to get full gdb functionality.

susestylin · December 8, 2011, 2:37am

I couldn’t find it in the core file with gdb, but as mentioned, gdb is a little complex.

Taking your suggestion, I ran strace on valid and failing sessions. What I found is a rogue attempt to open “/dev/tty” in the failing session:

open("/usr/NX/share/images/nxagent.xpm", O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such file or directory)
open("/usr/bin/nxagent.xpm", O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such file or directory)
open("/home/hook/bin/nxagent.xpm", O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such file or directory)
open("/usr/local/bin/nxagent.xpm", O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such file or directory)
open("/usr/bin/nxagent.xpm", O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such file or directory)
open("/bin/nxagent.xpm", O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such file or directory)
open("/usr/bin/X11/nxagent.xpm", O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such file or directory)
open("/usr/X11R6/bin/nxagent.xpm", O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such file or directory)
open("/usr/games/nxagent.xpm", O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such file or directory)
**open("/dev/tty", O_RDWR|O_NOCTTY|O_NONBLOCK) = -1 ENXIO (No such device or address)**
writev(2, {"*** glibc detected *** ", 23}, {"/usr/lib/NX/nxagent", 19}, {": ", 2}, {"free(): invalid pointer", 23}, {": 0x", 4}, {"084f0e48", 8}, {" ***
", 5}], 7) = 84

In a session that succeeds, the attempt to open /usr/games/nxagent.xpm is the last in the sequence. Now the question is to find out where this /dev/tty comes from.

susestylin · December 8, 2011, 3:37am

Just FYI, since the “gdb -c core” told me that the failing program was /usr/lib/NX/nxagent, I ran “strace -o /tmp/strace-nxagent /usr/lib/NX/nxagent” to generate the trace output.

djh-novell · December 8, 2011, 11:51am

susestylin wrote:
> Code:
> --------------------
> open("/usr/NX/share/images/nxagent.xpm", O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such file or directory)
> open("/usr/bin/nxagent.xpm", O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such file or directory)
> open("/home/hook/bin/nxagent.xpm", O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such file or directory)
> open("/usr/local/bin/nxagent.xpm", O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such file or directory)
> open("/usr/bin/nxagent.xpm", O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such file or directory)
> open("/bin/nxagent.xpm", O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such file or directory)
> open("/usr/bin/X11/nxagent.xpm", O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such file or directory)
> open("/usr/X11R6/bin/nxagent.xpm", O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such file or directory)
> open("/usr/games/nxagent.xpm", O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such file or directory)
> open("/dev/tty", O_RDWR|O_NOCTTY|O_NONBLOCK) = -1 ENXIO (No such device or address)
> writev(2, {"*** glibc detected *** “, 23}, {”/usr/lib/NX/nxagent", 19}, {": “, 2}, {“free(): invalid pointer”, 23}, {”: 0x", 4}, {“084f0e48”, 8}, {" ***
", 5}], 7) = 84
> --------------------
>
> In a session that succeeds, the attempt to open /usr/games/nxagent.xpm
> is the last in the sequence. Now the question is to find out where this
> /dev/tty comes from.

It’s great that you’re making progress. In a successful session, does it
succeed in opening /usr/games/nxagent.xpm or does it just stop trying at
that point?

It sounds like it may depend on whether it thinks it has a controlling
terminal or not. I guess reading the code is the next step.

robin_listas · December 8, 2011, 5:08pm

On 2011-12-08 02:36, susestylin wrote:
>> > so it is not that simple O:-)
> If you use “gdb -c core”, it will complain but will tell you “Core was
> generated by ‘/prog/xxx -opts’.”
>
> Once you know what program, then run “gdb -c core /prog/xxx” to get
> full gdb functionality.

Ah, I see. Yes, it says something:


Core was generated by `/usr/bin/nspluginwrapper -a -u'.
Program terminated with signal 11, Segmentation fault.
#0  0x00007fc7c2d57ccd in ?? ()

As to the second step, I can’t, the core is from last June and system has
changed in the time.

It is a core dump I found in my system one day not knowing where it came
from. So I was curious.

–
Cheers / Saludos,

Carlos E. R.
(from 11.4 x86_64 “Celadon” at Telcontar)

susestylin · December 10, 2011, 5:37am

No, the successful sessions do not open /usr/games/nxagent.xpm either, they also get “No such file or directory”.

susestylin · December 10, 2011, 6:08pm

I’ve downloaded the source for nxagent-3.4.0-9 from the OBS project (https://build.opensuse.org/package/files?package=NX&project=home%3Aplease_try_again) and I see that it pulls in the PATH environment variable, which is where all the prefixes for the nxagent.xpm paths appear to come from.

Bool nxagentMakeIcon(Display *display, Pixmap *nxIcon, Pixmap *nxMask)
{
  char *env_path = getenv("PATH");

What I’d like to do now is put a print/log statement in there to see what the value of env_path ends up being, but I have no idea how to go about compiling the code. There’s no configure or README file to tell me what to do.

susestylin · December 12, 2011, 1:03am

I’ve figured out how to compile it now. I used some info from the following pages:

The OBS project “NX.spec” spec file: (https://build.opensuse.org/package/view_file?file=NX.spec&package=NX&project=home%3Aplease_try_again&rev=1d141f31ef33c2f070f3a79dd5fe60be)
The OBS project “NX-3.4.0.patch” patch file: (https://build.opensuse.org/package/view_file?file=NX-3.4.0.patch&package=NX&project=home%3Aplease_try_again&rev=1d141f31ef33c2f070f3a79dd5fe60be)
A how-to-build from the NoMachine site: NoMachine NX - Support: Documentation

So, what I did was the following:

The NoMachine page said to download the components and extract them into a common folder (~/NX), so I did that, using the OBS packages (https://build.opensuse.org/package/files?package=NX&project=home%3Aplease_try_again) in preference to the NoMachine ones. When I couldn’t download a package from the OBS project, I used the "Sourcex: " link from the spec file to get it from NoMachine:

Version:        3.4.0
Source4:        http://web04.nomachine.com/download/%{version}/sources/nx-X11-%{version}-3.tar.gz

 ~/NX # wget http://web04.nomachine.com/download/3.4.0/sources/nx-X11-3.4.0-3.tar.gz
 ~/NX # tar -jxvf nx-X11-3.4.0-3.tar.gz

The OBS project differs a little from the NoMachine packages in that it doesn’t implement the nxauth component. I then looked at the OBS .patch file which showed some changes related to removing nxauth from the compile. So I put the .patch file into ~/NX and patched the extracted tarballs using patch:

 ~/NX # patch -p1 -i NX-3.4.0.patch

Before I could compile, I needed to install some dependencies. The .spec file helped again here, I just “zypper in” the missing packages listed in the “BuildRequires:” line:

BuildRequires:  gcc-c++ libjpeg62-devel libpng12-compat-devel libopenssl-devel xorg-x11-proto-devel xorg-x11-util-devel

 ~/NX # zypper in gcc-c++ libjpeg62-devel libpng12-compat-devel libopenssl-devel xorg-x11-proto-devel
 xorg-x11-util-devel

The NoMachine documentation then said to run “make World” in the ~/NX/nx-X11/ directory. Lo and behold, a bunch of output later, I’d compiled my own NX package:

 ~NX/nx-X11 # make World
...

Full build of Release 6.9 complete.

 ~/NX/nx-X11 #

susestylin · December 20, 2011, 3:46am

The problem seems related to dirty memory being allocated for strings in the code. Once I explicitly added code to null out the full value of the strings after allocation, the problem went away, for new and old users.

Bool nxagentMakeIcon(Display *display, Pixmap *nxIcon, Pixmap *nxMask)
{
  char *env_path = getenv("PATH");
  int lenght_env_path = 0;
  char icon_filename [256];
  char default_path [256];
  char *icon_path = malloc( strlen(env_path) + sizeof(icon_filename) );
  FILE *icon_fp;

  if (env_path == NULL)
    lenght_env_path = 0;
  else
    lenght_env_path = strlen(env_path) + 1;
  strncpy(icon_filename, "", 255);
  strncpy(default_path, "", 255);
  strncpy(icon_path, "", strlen(env_path) + sizeof(icon_filename) );

  strcat(icon_filename, NXAGENT_ICON_NAME);
  strcat(default_path,"/usr/NX/share/images/");
  strcat(default_path,icon_filename);

  if ((icon_fp = fopen(default_path, "r")) == NULL)
  {
    char *s;
    char *temp_path = malloc(lenght_env_path + strlen(icon_filename) );
    char *temp_path1 = malloc(lenght_env_path + strlen(icon_filename) );

    strncpy(temp_path, env_path, strlen(env_path));
    strncpy(temp_path1, "", lenght_env_path + strlen(icon_filename) );

    ...

    free(temp_path);
    free(temp_path1);

...

I added some output to the code and re-compiled. When memory is dirty, the variable temp_path (which is not nulled in the original code), has some ugly contents, which then messes up the strncpy from env_path. It is my assumption that it is the garbage that messes up the free(temp_path) later in the code.

nxagentMakeIcon: env_path=/usr/bin:/home/hook/bin:/usr/local/bin:/usr/bin:/bi
n:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/jvm/jre/bin
displayMemory:
BFDF4B5B | 2F 75 73 72 2F 62 69 6E 3A 2F 68 6F 6D 65 2F 68 | /usr/bin:/home/h
BFDF4B6B | 6F 6F 6B 2F 62 69 6E 3A 2F 75 73 72 2F 6C 6F 63 | ook/bin:/usr/loc
BFDF4B7B | 61 6C 2F 62 69 6E 3A 2F 75 73 72 2F 62 69 6E 3A | al/bin:/usr/bin:
BFDF4B8B | 2F 62 69 6E 3A 2F 75 73 72 2F 62 69 6E 2F 58 31 | /bin:/usr/bin/X1
BFDF4B9B | 31 3A 2F 75 73 72 2F 58 31 31 52 36 2F 62 69 6E | 1:/usr/X11R6/bin
BFDF4BAB | 3A 2F 75 73 72 2F 67 61 6D 65 73 3A 2F 75 73 72 | :/usr/games:/usr
BFDF4BBB | 2F 6C 69 62 2F 6A 76 6D 2F 6A 72 65 2F 62 69 6E | /lib/jvm/jre/bin
nxagentMakeIcon: initial temp_path=ºººººººººººººººººººººººººººººººººººººººººº
ººººººººººººººººººººººººººººººººººººººººººººººººººººººººººººººººººººººººººººº
ººººº^KEEEEEE^G<89>
displayMemory:
08561E48 | BA BA BA BA BA BA BA BA BA BA BA BA BA BA BA BA | ºººººººººººººººº
08561E58 | BA BA BA BA BA BA BA BA BA BA BA BA BA BA BA BA | ºººººººººººººººº
08561E68 | BA BA BA BA BA BA BA BA BA BA BA BA BA BA BA BA | ºººººººººººººººº
08561E78 | BA BA BA BA BA BA BA BA BA BA BA BA BA BA BA BA | ºººººººººººººººº
08561E88 | BA BA BA BA BA BA BA BA BA BA BA BA BA BA BA BA | ºººººººººººººººº
08561E98 | BA BA BA BA BA BA BA BA BA BA BA BA BA BA BA BA | ºººººººººººººººº
08561EA8 | BA BA BA BA BA BA BA BA BA BA BA BA BA BA BA BA | ºººººººººººººººº
08561EB8 | BA BA BA BA BA BA BA BA BA BA BA BA 0B 45 45 45 | ºººººººººººº.EEE
08561EC8 | 45 45 45 07 89 __ __ __ __ __ __ __ __ __ __ __ | EEE.<89>
nxagentMakeIcon: initial temp_path1=ººººººººººººººººººººººººººººººººººººººººº
ººººººººººººººººººººººººººººººººººººººººººººººººººººººººººººººººººººººººººººº
ºººººº^ZEEEEEE^G9^L
displayMemory:
08561ED0 | BA BA BA BA BA BA BA BA BA BA BA BA BA BA BA BA | ºººººººººººººººº
08561EE0 | BA BA BA BA BA BA BA BA BA BA BA BA BA BA BA BA | ºººººººººººººººº
08561EF0 | BA BA BA BA BA BA BA BA BA BA BA BA BA BA BA BA | ºººººººººººººººº
08561F00 | BA BA BA BA BA BA BA BA BA BA BA BA BA BA BA BA | ºººººººººººººººº
08561F10 | BA BA BA BA BA BA BA BA BA BA BA BA BA BA BA BA | ºººººººººººººººº
08561F20 | BA BA BA BA BA BA BA BA BA BA BA BA BA BA BA BA | ºººººººººººººººº
08561F30 | BA BA BA BA BA BA BA BA BA BA BA BA BA BA BA BA | ºººººººººººººººº
08561F40 | BA BA BA BA BA BA BA BA BA BA BA BA 1A 45 45 45 | ºººººººººººº.EEE
08561F50 | 45 45 45 07 39 0C __ __ __ __ __ __ __ __ __ __ | EEE.9.
nxagentMakeIcon: initial env_path temp_path=/usr/bin:/home/hook/bin:/usr/loca
l/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/jvm/jre/b
inºººººººººººº^KEEEEEE^G<89>
displayMemory:
08561E48 | 2F 75 73 72 2F 62 69 6E 3A 2F 68 6F 6D 65 2F 68 | /usr/bin:/home/h
08561E58 | 6F 6F 6B 2F 62 69 6E 3A 2F 75 73 72 2F 6C 6F 63 | ook/bin:/usr/loc
08561E68 | 61 6C 2F 62 69 6E 3A 2F 75 73 72 2F 62 69 6E 3A | al/bin:/usr/bin:
08561E78 | 2F 62 69 6E 3A 2F 75 73 72 2F 62 69 6E 2F 58 31 | /bin:/usr/bin/X1
08561E88 | 31 3A 2F 75 73 72 2F 58 31 31 52 36 2F 62 69 6E | 1:/usr/X11R6/bin
08561E98 | 3A 2F 75 73 72 2F 67 61 6D 65 73 3A 2F 75 73 72 | :/usr/games:/usr
08561EA8 | 2F 6C 69 62 2F 6A 76 6D 2F 6A 72 65 2F 62 69 6E | /lib/jvm/jre/bin
08561EB8 | BA BA BA BA BA BA BA BA BA BA BA BA 0B 45 45 45 | ºººººººººººº.EEE
08561EC8 | 45 45 45 07 89 __ __ __ __ __ __ __ __ __ __ __ | EEE.<89>
nxagentMakeIcon: initial nulled temp_path1=
displayMemory:
08561ED0 |

The code above also shows temp_path1, which is nulled in the original code.

When I attach a debugger (valgrind), it changes the memory allocation (you can see the address change below) and the memory is clean and the NX session succeeds even for the old users.

nxagentMakeIcon: env_path=/usr/bin:/home/hook/bin:/usr/local/bin:/usr/bin:/bi
n:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/jvm/jre/bin
displayMemory:
BEEACADB | 2F 75 73 72 2F 62 69 6E 3A 2F 68 6F 6D 65 2F 68 | /usr/bin:/home/h
BEEACAEB | 6F 6F 6B 2F 62 69 6E 3A 2F 75 73 72 2F 6C 6F 63 | ook/bin:/usr/loc
BEEACAFB | 61 6C 2F 62 69 6E 3A 2F 75 73 72 2F 62 69 6E 3A | al/bin:/usr/bin:
BEEACB0B | 2F 62 69 6E 3A 2F 75 73 72 2F 62 69 6E 2F 58 31 | /bin:/usr/bin/X1
BEEACB1B | 31 3A 2F 75 73 72 2F 58 31 31 52 36 2F 62 69 6E | 1:/usr/X11R6/bin
BEEACB2B | 3A 2F 75 73 72 2F 67 61 6D 65 73 3A 2F 75 73 72 | :/usr/games:/usr
BEEACB3B | 2F 6C 69 62 2F 6A 76 6D 2F 6A 72 65 2F 62 69 6E | /lib/jvm/jre/bin
nxagentMakeIcon: initial temp_path=
displayMemory:
04A63188 |
nxagentMakeIcon: initial temp_path1=
displayMemory:
04A63238 |
nxagentMakeIcon: initial env_path temp_path=/usr/bin:/home/hook/bin:/usr/loca
l/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/jvm/jre/b
in
displayMemory:
04A63188 | 2F 75 73 72 2F 62 69 6E 3A 2F 68 6F 6D 65 2F 68 | /usr/bin:/home/h
04A63198 | 6F 6F 6B 2F 62 69 6E 3A 2F 75 73 72 2F 6C 6F 63 | ook/bin:/usr/loc
04A631A8 | 61 6C 2F 62 69 6E 3A 2F 75 73 72 2F 62 69 6E 3A | al/bin:/usr/bin:
04A631B8 | 2F 62 69 6E 3A 2F 75 73 72 2F 62 69 6E 2F 58 31 | /bin:/usr/bin/X1
04A631C8 | 31 3A 2F 75 73 72 2F 58 31 31 52 36 2F 62 69 6E | 1:/usr/X11R6/bin
04A631D8 | 3A 2F 75 73 72 2F 67 61 6D 65 73 3A 2F 75 73 72 | :/usr/games:/usr
04A631E8 | 2F 6C 69 62 2F 6A 76 6D 2F 6A 72 65 2F 62 69 6E | /lib/jvm/jre/bin
nxagentMakeIcon: initial nulled temp_path1=
displayMemory:
04A63238 |

Once I add a statement to null the value of temp_path (similar to the one for temp_path1) immediately after allocation (before the strncpy from env_path), the problem goes away and NX sessions for new and old users succeed.


    strncpy(temp_path, "", lenght_env_path + strlen(icon_filename) );
    strncpy(temp_path, env_path, strlen(env_path));
    strncpy(temp_path1, "", lenght_env_path + strlen(icon_filename) );