Well, with all the interest in getting R5 up and running under Solaris
2.x, I am here to announce success!!

First, these patches will compile R5 using gcc 2.3.2 (shared libraries
and all) and do not require anything from /usr/ucb*.  (signals have
left me shaking to badly to tackle getting rid of the socket library
stuff).

Next, the directions:

1) obtain the patches in the file X11r5-Solaris2-patches.tar.Z on
export and apply them (credit where credit is REALLY due -- The author
of those patches did 90% of the work -- His/Her name is not included
in the README file so I can't give thanks properly).

2) grab the sun.cf file below and modify your's to match it (or just
use it as is).

3) add HasCygnus to your site.def file in the "before" section

4) apply the patches below.  Things should compile and load fine.

This has only been tested on a cg3.  From the previous patches, bw2
should work, but from GOTCHA(2) I can't speak for cg6.  I've been
running the cg3 server for the last couple of weeks, with no problems
(I've also done quite a few of the contributed clients).

GOTCHAS:

1) for reasons that I don't understand (bug, feature,
misunderstanding?), allowing xinit to set SIGUSR1 to SIG_IGN after the
fork causes xinit to tank (leaving the server running quite nicely).
Removing this stopped the core dumps, and didn't seem to have any
effect on xinit or Xsun.

2) Since I didn't have a binary that would link, I hand to comment out
the cg6 init function call.  This may break the cg6 server (I don't
know, I don't have one).  Maybe some kind Sun person can provide us
with a Solaris .o file for it.

HINTS:

If you store your X libraries in a non-standard place (like we do),
you can build your clients with the environment variable LD_RUN_PATH
set to the location of your libraries, and users will not have to use
LD_LIBRARY_PATH when running them.  For my make (using sh/ksh), I did:
	LD_RUN_PATH=/usr/local/pkg/X11/lib gnumake World


Dana Chee
dana@thumper.bellcore.com

P.S.  Thanks to the Sun people reading comp.os.solaris and the folks
at Cygnus for helping me get all this working!

==================== sun.cf
XCOMM platform:  $XConsortium: sun.cf,v 1.72.1.1 92/03/18 13:13:37 rws Exp $

#define OSName            SunOS 5.1
XCOMM operating system:  OSName
#define OSMajorVersion    5
#define OSMinorVersion    1
#define OSTeenyVersion    0

#if OSMajorVersion < 4 || (OSMajorVersion == 4 && OSMinorVersion < 1)
/* We do not guarantee this will work */
#define BootstrapCFlags   -DNOSTDHDRS
#define StandardDefines   -DNOSTDHDRS
#endif
#if OSMajorVersion == 5
#define BootstrapCFlags   -DSVR4
#define StandardDefines   -DSVR4
#endif

/* You ALSO need this if you have Sun ld patch 100170-06 or later to 4.1.1 */
#if OSMajorVersion == 4 && (OSMinorVersion > 1 || (OSMinorVersion == 1 && OSTeenyVersion > 1))
#define SunPost411FCSLd	  YES
#endif

#define HasSaberC	  YES
#define HasNdbm		  YES
#define HasShm		  YES
#define HasSecureRPC	  YES
#define SetTtyGroup	  YES
#define HasPutenv	  YES
#define XsunServer	  YES		/* has color and mono support */
#define XsunMonoServer	  YES		/* monochrome only */
#define BuildPexExt	  XsunServer
#define InstKmemFlags	  -g kmem -m 2755

#ifdef sparc
#undef sparc
#define SparcArchitecture
#endif
#ifdef sun386
#undef sun386
#define i386Architecture
#endif

#if OSMajorVersion == 5
#define CppCmd			/usr/ccs/lib/cpp
#define SystemV4		YES
#define HasSockets		YES
#define ExtraLibraries		-lsocket -lnsl
#if HasCygnus
#define PositionIndependentCFlags  -fPIC
#endif
#include 
#else
#include 
#endif

#if HasGcc

#define SharedLibraryCcCmd cc
#define ExtraLoadFlags -B/usr/bin/
#define AllocateLocalDefines /**/

.c.o:
	$(CC) -c $(CFLAGS) $*.c

#else

#if ! HasCygnus
#define AllocateLocalDefines -DINCLUDE_ALLOCA_H
#define SharedAllocateLocalDefines -DINCLUDE_ALLOCA_H
#endif

#ifdef mc68000
#define DefaultCCOptions -f68881 -pipe
#else
#define DefaultCCOptions -pipe
#endif

#define PexCCOptions DefaultCCOptions

#endif

#define LibraryCCOptions /* don't want special floating point */

#ifndef UseSunWindowsInServer
#define UseSunWindowsInServer	NO	/* link in SunWindows support? */
#endif

#if UseSunWindowsInServer
#define SunWindowsDefines -DSUN_WINDOWS
#else
#define SunWindowsDefines /* as nothing */
#endif

#define ServerOSDefines SunWindowsDefines AllocateLocalDefines

==================== patches
*** ./config/imakemdep.h.chee1	Sat Nov 21 17:45:20 1992
--- ./config/imakemdep.h	Sat Nov 21 17:56:32 1992
***************
*** 126,131 ****
--- 126,134 ----
  #if defined(_IBMR2) && !defined(DEFAULT_CPP)
  #define DEFAULT_CPP "/usr/lpp/X11/Xamples/util/cpp/cpp"
  #endif
+ #if defined(sun) && defined(SVR4)
+ #define DEFAULT_CPP "/usr/ccs/lib/cpp"
+ #endif
  
  /*
   * Step 5:  cpp_argv
*** ./server/ddx/sun/sunKbd.c.chee1	Sun Nov 22 07:41:01 1992
--- ./server/ddx/sun/sunKbd.c	Thu Dec  3 16:01:46 1992
***************
*** 57,63 ****
  #include 
  
  #ifdef SVR4
! #include 
  #endif
  
  typedef struct {
--- 57,66 ----
  #include 
  
  #ifdef SVR4
! #include        
! #include        
! #include        
! #include        
  #endif
  
  typedef struct {
***************
*** 184,191 ****
  		    (void) close( kbdFd );
  		    sysKbPriv.fd = -1;
  		} else {
  		    if (fcntl (kbdFd, F_SETFL, (FNDELAY|FASYNC)) < 0
! 			|| fcntl(kbdFd, F_SETOWN, getpid()) < 0) {
  			perror("sunKbdProc");
  			FatalError("Can't set up kbd on fd %d\n", kbdFd);
  		    }
--- 187,199 ----
  		    (void) close( kbdFd );
  		    sysKbPriv.fd = -1;
  		} else {
+ #ifdef SVR4
+ 		    if (fcntl(kbdFd, F_SETFL, (O_NONBLOCK|O_SYNC)) < 0
+ 			|| ioctl(kbdFd, I_SETSIG, S_INPUT) < 0) {
+ #else
  		    if (fcntl (kbdFd, F_SETFL, (FNDELAY|FASYNC)) < 0
! 		    	|| fcntl(kbdFd, F_SETOWN, getpid()) < 0) {
! #endif
  			perror("sunKbdProc");
  			FatalError("Can't set up kbd on fd %d\n", kbdFd);
  		    }
***************
*** 590,596 ****
  
  sunEnqueueAutoRepeat ()
  {
!     int	oldmask;
      int	delta;
  
      if (sysKbPriv.ctrl->autoRepeat != AutoRepeatModeOn) {
--- 598,606 ----
  
  sunEnqueueAutoRepeat ()
  {
! #ifndef SVR4
!     int oldmask;
! #endif
      int	delta;
  
      if (sysKbPriv.ctrl->autoRepeat != AutoRepeatModeOn) {
***************
*** 611,621 ****
--- 621,639 ----
       */
      autoRepeatEvent.u.keyButtonPointer.time += delta;
      autoRepeatEvent.u.u.type = KeyRelease;
+ #ifdef SVR4
+     sighold(SIGIO);
+ #else
      oldmask = sigblock (sigmask(SIGIO));
+ #endif
      mieqEnqueue (&autoRepeatEvent);
      autoRepeatEvent.u.u.type = KeyPress;
      mieqEnqueue (&autoRepeatEvent);
+ #ifdef SVR4
+     sigrelse(SIGIO);
+ #else
      sigsetmask (oldmask);
+ #endif
      /* Update time of last key down */
      tvplus(autoRepeatLastKeyDownTv, autoRepeatLastKeyDownTv, 
  		    autoRepeatDeltaTv);
***************
*** 645,651 ****
--- 663,674 ----
      int 	kbdFd;
      int 	tmp;
      int		kbdOpenedHere;
+ #ifdef SVR4
+     sigset_t	hold_mask;
+     sigset_t	old_mask;
+ #else
      int		old_mask;
+ #endif
      int		toread;
      char	junk[8192];
  
***************
*** 658,664 ****
--- 681,692 ----
       * Workaround for SS1 serial driver kernel bug when KIOCTRANS ioctl()s
       * occur too closely together in time.
       */
+ #ifdef SVR4
+     sigfillset(&hold_mask);
+     sigprocmask(SIG_BLOCK, &hold_mask, &old_mask);
+ #else
      old_mask = sigblock (~0);
+ #endif
      gettimeofday(&tv, (struct timezone *) NULL);
      tvminus(lastChngKbdDeltaTv, tv, lastChngKbdTransTv);
      lastChngKbdDelta = TVTOMILLI(lastChngKbdDeltaTv);
***************
*** 741,753 ****
--- 769,789 ----
      }
      if ( kbdOpenedHere )
  	(void) close( kbdFd );
+ #ifdef SVR4
+     sigprocmask(SIG_SETMASK, &old_mask, NULL);
+ #else
      sigsetmask (old_mask);
+ #endif
      return(0);
  
  bad:
      if ( kbdOpenedHere && kbdFd >= 0 )
  	(void) close( kbdFd );
+ #ifdef SVR4
+     sigprocmask(SIG_SETMASK, &old_mask, NULL);
+ #else
      sigsetmask (old_mask);
+ #endif
      return(-1);
  }
  
***************
*** 921,923 ****
--- 957,969 ----
  	autoRepeatReady = 0;
      }
  }
+ 
+ #ifdef SVR4
+ int
+ usleep(usec)
+     unsigned long usec;
+ {
+     poll((struct poll *) 0, (size_t) 0, usec / 1000);	/* ms resolution */
+ }
+ #endif
+ 
*** ./server/ddx/sun/sunBW2.c.chee1	Sun Nov 22 08:06:46 1992
--- ./server/ddx/sun/sunBW2.c	Sun Nov 22 08:08:11 1992
***************
*** 155,161 ****
--- 155,165 ----
       * rumors that this is a good idea and that it shouldn't
       * hurt anything.
       */
+ #ifdef SVR4
+     pagemask = sysconf(_SC_PAGESIZE) - 1;
+ #else
      pagemask = getpagesize() - 1;
+ #endif
      mapsize = (fbType.fb_size + pagemask) & ~pagemask;
      addr = 0;
  
*** ./server/ddx/sun/sunCG6C.c.chee1	Sun Nov 22 08:17:36 1992
--- ./server/ddx/sun/sunCG6C.c	Sun Nov 22 08:22:54 1992
***************
*** 91,99 ****
--- 91,101 ----
  	return FALSE;
  
  #ifdef sparc
+ #ifdef notdef
      if (!sunGXInit (pScreen, &sunFbs[index]))
  	return FALSE;
  #endif
+ #endif
  
      if (!cfbFinishScreenInit(pScreen, pbits, w, h,
  			monitorResolution, monitorResolution, w))
***************
*** 144,150 ****
--- 146,156 ----
  				 argc, argv)) < 0)
  	return FALSE;
  
+ #ifdef SVR4
+     pagemask = sysconf(_SC_PAGESIZE) - 1;
+ #else
      pagemask = getpagesize() - 1;
+ #endif
      imagelen = CG6_IMAGEOFF + fbType.fb_width * fbType.fb_height;
      mapsize = (imagelen + pagemask) & ~pagemask;
      addr = 0;
*** ./server/ddx/sun/sunMouse.c.chee1	Sun Nov 22 08:23:08 1992
--- ./server/ddx/sun/sunMouse.c	Thu Dec  3 15:55:13 1992
***************
*** 49,55 ****
  #endif
  
  #ifdef SVR4
! #define	sigmask(m)	(1 <<((m)-1))
  #endif
  
  #define NEED_EVENTS
--- 49,58 ----
  #endif
  
  #ifdef SVR4
! #include	
! #include	
! #include	
! #include	
  #endif
  
  #define NEED_EVENTS
***************
*** 134,145 ****
  			return (!Success);
  		    }
  #ifdef SVR4
! 		    if (fcntl (fd, F_SETFL, (FNDELAY|FASYNC)) < 0)
  #else
  		    if (fcntl (fd, F_SETFL, (FNDELAY|FASYNC)) < 0
! 			|| fcntl(fd, F_SETOWN, getpid()) < 0)
  #endif
- 								{
  			    perror("sunMouseProc");
  			    ErrorF("Can't set up mouse on fd %d\n", fd);
  			}
--- 137,148 ----
  			return (!Success);
  		    }
  #ifdef SVR4
! 		    if (fcntl (fd, F_SETFL, (O_NONBLOCK | O_SYNC)) < 0
! 			|| ioctl(fd, I_SETSIG, S_INPUT) < 0) {
  #else
  		    if (fcntl (fd, F_SETFL, (FNDELAY|FASYNC)) < 0
! 			|| fcntl(fd, F_SETOWN, getpid()) < 0) {
  #endif
  			    perror("sunMouseProc");
  			    ErrorF("Can't set up mouse on fd %d\n", fd);
  			}
***************
*** 438,448 ****
--- 441,461 ----
      ScreenPtr	pScreen;
      int		x, y;
  {
+ #ifndef SVR4
      int	    oldmask;
+ #endif
  
+ #ifdef SVR4
+     sighold(SIGIO);
+ #else
      oldmask = sigblock (sigmask(SIGIO));
+ #endif
      miPointerWarpCursor (pScreen, x, y);
+ #ifdef SVR4
+     sigrelse(SIGIO);
+ #else
      sigsetmask (oldmask);
+ #endif
  }
  
  #ifdef SUN_WINDOWS
*** ./server/ddx/sun/sunInit.c.chee1	Sun Nov 22 09:00:55 1992
--- ./server/ddx/sun/sunInit.c	Wed Dec  2 09:30:35 1992
***************
*** 107,114 ****
--- 107,131 ----
      sunEnqueueEvents ();
  }
  
+ #ifdef SVR4
+ 
  sunFbDataRec sunFbData[] = {
  #ifndef MONO_ONLY
+     sunBW2Probe,  	"/dev/fbs/bwtwo0",	    sunBW2Create,
+     sunCG3CProbe,  	"/dev/fbs/cgthree0",    sunCG3CCreate,
+ #ifdef FBTYPE_SUNFAST_COLOR
+     sunCG6CProbe,	"/dev/fbs/cgsix0",	    sunCG6CCreate,
+ #endif
+     sunCG2CProbe,  	"/dev/fbs/cgtwo0",	    sunCG2CCreate,
+     sunCG4CProbe,  	"/dev/fbs/cgfour0",	    sunCG4CCreate,
+ #endif
+     sunBW2Probe,  	"/dev/fbs/bwtwo0",	    sunBW2Create,
+ };
+ 
+ #else
+ 
+ sunFbDataRec sunFbData[] = {
+ #ifndef MONO_ONLY
      sunBW2Probe,  	"/dev/bwtwo0",	    sunBW2Create,
      sunCG3CProbe,  	"/dev/cgthree0",    sunCG3CCreate,
  #ifdef FBTYPE_SUNFAST_COLOR
***************
*** 119,124 ****
--- 136,142 ----
  #endif
      sunBW2Probe,  	"/dev/bwtwo0",	    sunBW2Create,
  };
+ #endif
  
  #ifdef MONO_ONLY
  #define DEV_START   0
***************
*** 284,289 ****
--- 302,310 ----
  {
      DevicePtr p, k;
      static int  zero = 0;
+ #ifdef SVR4
+     struct sigaction	a;
+ #endif
      
      p = AddInputDevice(sunMouseProc, TRUE);
      k = AddInputDevice(sunKbdProc, TRUE);
***************
*** 296,302 ****
--- 317,330 ----
      miRegisterPointerDevice(screenInfo.screens[0], p);
      if (!mieqInit (k, p))
  	return FALSE;
+ #ifdef SVR4
+     a.sa_handler = SigIOHandler;
+     a.sa_flags = NULL;
+     sigemptyset(&a.sa_mask);
+     sigaction(SIGIO, &a, NULL);
+ #else
      signal(SIGIO, SigIOHandler);
+ #endif
  }
  
  
*** ./server/os/rpcauth.c.chee1	Sun Nov 22 08:47:16 1992
--- ./server/os/rpcauth.c	Sun Nov 22 08:47:55 1992
***************
*** 74,80 ****
          why = AUTH_TOOWEAK;
          goto bad2;
      }
!     if ((why = _authenticate(&r, &msg)) != AUTH_OK) {
              goto bad2;
      }
      return (((struct authdes_cred *) r.rq_clntcred)->adc_fullname.name); 
--- 74,80 ----
          why = AUTH_TOOWEAK;
          goto bad2;
      }
!     if ((why = __authenticate(&r, &msg)) != AUTH_OK) {
              goto bad2;
      }
      return (((struct authdes_cred *) r.rq_clntcred)->adc_fullname.name); 
*** ./clients/xinit/xinit.c.chee1	Wed Dec  2 09:30:52 1992
--- ./clients/xinit/xinit.c	Tue Dec  8 16:23:44 1992
***************
*** 142,148 ****
  SIGVAL sigAlarm(sig)
  	int sig;
  {
! #ifdef SYSV
  	signal (sig, sigAlarm);
  #endif
  }
--- 142,148 ----
  SIGVAL sigAlarm(sig)
  	int sig;
  {
! #if defined(SYSV) || defined(SVR4)
  	signal (sig, sigAlarm);
  #endif
  }
***************
*** 151,157 ****
  sigUsr1(sig)
  	int sig;
  {
! #ifdef SYSV
  	signal (sig, sigUsr1);
  #endif
  }
--- 151,157 ----
  sigUsr1(sig)
  	int sig;
  {
! #if defined(SYSV) || defined(SVR4)
  	signal (sig, sigUsr1);
  #endif
  }
***************
*** 414,420 ****
  		 * will notice this and send SIGUSR1 back
  		 * at xinit when ready to accept connections
  		 */
! 		(void) signal(SIGUSR1, SIG_IGN);
  		/*
  		 * prevent server from getting sighup from vhangup()
  		 * if client is xterm -L
--- 414,420 ----
  		 * will notice this and send SIGUSR1 back
  		 * at xinit when ready to accept connections
  		 */
! 		/*(void) signal(SIGUSR1, SIG_IGN);*/
  		/*
  		 * prevent server from getting sighup from vhangup()
  		 * if client is xterm -L
*** ./clients/xmag/xmag.c.chee1	Tue Oct  1 09:39:36 1991
--- ./clients/xmag/xmag.c	Wed Dec  2 13:20:14 1992
***************
*** 888,894 ****
  
  
  
! static Widget pane1, pane2, pane3, cclose, replace, new, select, paste, label;
  
  /*
   * PopupNewScale() -- Create and popup a new scale composite.
--- 888,894 ----
  
  
  
! static Widget pane1, pane2, pane3, cclose, replace, new, xselect, paste, label;
  
  /*
   * PopupNewScale() -- Create and popup a new scale composite.
***************
*** 917,925 ****
    new = XtCreateManagedWidget("new", commandWidgetClass, pane2,
  			      (Arg *) NULL, 0);
    XtAddCallback(new, XtNcallback, NewCB, (XtPointer)NULL);
!   select = XtCreateManagedWidget("select", commandWidgetClass, pane2,
  			      (Arg *) NULL, 0);
!   XtAddCallback(select, XtNcallback, SelectCB, (XtPointer)data);
    paste = XtCreateManagedWidget("paste", commandWidgetClass, pane2,
  			      (Arg *) NULL, 0);
    XtAddCallback(paste, XtNcallback, PasteCB, (XtPointer)data);
--- 917,925 ----
    new = XtCreateManagedWidget("new", commandWidgetClass, pane2,
  			      (Arg *) NULL, 0);
    XtAddCallback(new, XtNcallback, NewCB, (XtPointer)NULL);
!   xselect = XtCreateManagedWidget("select", commandWidgetClass, pane2,
  			      (Arg *) NULL, 0);
!   XtAddCallback(xselect, XtNcallback, SelectCB, (XtPointer)data);
    paste = XtCreateManagedWidget("paste", commandWidgetClass, pane2,
  			      (Arg *) NULL, 0);
    XtAddCallback(paste, XtNcallback, PasteCB, (XtPointer)data);

-- 
+*************************************************************************+
*  Dana Chee				(201) 829-4488			  *
*  Bellcore								  *
*  Room 2P-298								  *
*  445 South Street			ARPA: dana@bellcore.com		  *
*  Morristown,  NJ  07960-1910		UUCP: {gateways}!bellcore!dana	  *
+*************************************************************************+