tooltalk: Fix bad assumptions about sizeof(uid_t)

In part of the tooltalk rpc code (mp_message.c), it was assumed that on
the majority of platforms, sizeof(uid_t)=sizeof(gid_t)=sizeof(long).  On
Linux-x64, uid_t is an unsigned int, which makes the code fail: all
tooltalk messages fail to send with an RPC_CANTENCODEARGS at the
rpc-level, and TT_INTERNAL_ERR for the actual program.  We instead
change the code to explicitly examine sizeof(uid_t) to see whether it is
int or long sized. This allows tooltalk-dependent functinoality
like logout and multiple calls to dtfile to work.

diff --git a/cde/lib/tt/lib/mp/mp_message.C b/cde/lib/tt/lib/mp/mp_message.C
index 1c00fca69b60b03f5443e4f34636bf1b6603ba27..06afb24539697b1d218fc4d920a3e51e2a2e7ebe 100644 (file)
--- a/cde/lib/tt/lib/mp/mp_message.C
+++ b/cde/lib/tt/lib/mp/mp_message.C
@@ -47,7 +47,7 @@
 #include <mp/mp_procid.h>
 #include <mp/mp_session.h>
 #include <util/tt_enumname.h>
-
+#include <util/tt_port.h>
        
 //
 // Base constructor for a message.  ::xdr() relies on this
@@ -302,33 +302,44 @@ xdr(XDR *xdrs)
        // Even though in Solaris 2.x uid_t and gid_t are typedef'd as
        // u_long, the compiler complains if they are not explicitly
        // coerced.
+
+       /*
+        * Obviously there is no xdr_u_uid_t, so we have to hack around this.
+        * At least by using sizeof checks, we aren't manually specifying
+        * the size of uid_t depending on the platform (which broke terribly
+        * when we tried to port to 64-bit linux at first).
+        * 
+        * A side-effect of testing this way is that we have to spell out the pointer
+        * casts, since one of them is actually wrong for any given platform (but
+        * is also a dead code branch)
+        */
        if (_ptr_guards&_TT_MSK_UID) {
                if (xdrs->x_op == XDR_DECODE) {
                        _uid = 0;
                }
-#ifdef __osf__
-               if (! xdr_u_int(xdrs, &_uid)) {
-                       return(0);
-               }
-#else
-               if (! xdr_u_long(xdrs,(u_long *) &_uid)) {
-                       return(0);
+               if (sizeof (int) == sizeof(uid_t)) {
+                       if (! xdr_u_int(xdrs,(u_int *) &_uid)) return 0;
+               } else if (sizeof (long) == sizeof(uid_t)) {
+                       if (! xdr_u_long(xdrs,(u_long *) &_gid)) return 0;
+               } else {
+                       _tt_syslog( 0, LOG_ERR, "_Tt_message::xdr(XDR*): uid_t size is not equal to int or long; TOOLTALK RPC WILL NOT WORK!" );
                }
-#endif
        }
        if (_ptr_guards&_TT_MSK_GID) {
                if (xdrs->x_op == XDR_DECODE) {
                        _gid = 0;
                }
-#ifdef __osf__
-               if (! xdr_u_int(xdrs, &_gid)) {
-                       return(0);
-               }
-#else
-               if (! xdr_u_long(xdrs,(u_long *) &_gid)) {
-                       return(0);
+               if (sizeof (int) == sizeof(gid_t)) {
+                       if (! xdr_u_int(xdrs,(u_int *) &_gid)) {
+                               return(0);
+                       }
+               } else if (sizeof (long) == sizeof(gid_t)) {
+                       if (! xdr_u_long(xdrs,(u_long *) &_gid)) {
+                               return(0);
+                       }
+               } else {
+                       _tt_syslog( 0, LOG_ERR, "_Tt_message::xdr(XDR*): gid_t size is not equal to int or long; TOOLTALK RPC WILL NOT WORK!" );
                }
-#endif
        }
 
        if (_ptr_guards&_TT_MSK_SESSION) {