Fix integer overflow when allocating large pixmaps.

(Freedesktop.org bug #594)

Obtained from:	Red Hat via simon
Security:	CAN-2005-2495

2 files changed, 170 insertions, 1 deletions

diff --git a/x11-servers/xorg-server/Makefile b/x11-servers/xorg-server/Makefile
index afeabd2d5f80..5b2628813f4a 100644
--- a/x11-servers/xorg-server/Makefile
+++ b/x11-servers/xorg-server/Makefile
@@ -7,7 +7,7 @@
 
 PORTNAME=	xorg-server
 PORTVERSION=	6.8.2
-PORTREVISION=	4
+PORTREVISION=	5
 CATEGORIES=	x11-servers
 MASTER_SITES=	${MASTER_SITE_XORG}
 MASTER_SITE_SUBDIR=	X11R${PORTVERSION}/src
diff --git a/x11-servers/xorg-server/files/patch-CAN-2005-2495 b/x11-servers/xorg-server/files/patch-CAN-2005-2495
new file mode 100644
index 000000000000..e9b7ded09739
--- /dev/null
+++ b/x11-servers/xorg-server/files/patch-CAN-2005-2495
@@ -0,0 +1,169 @@
+--- programs/Xserver/afb/afbpixmap.c.orig	Fri Apr 23 20:59:39 2004
++++ programs/Xserver/afb/afbpixmap.c	Tue Sep  6 17:08:01 2005
+@@ -73,10 +73,14 @@ afbCreatePixmap(pScreen, width, height, 
+ 	int				depth;
+ {
+ 	PixmapPtr pPixmap;
+-	int datasize;
+-	int paddedWidth;
++	size_t datasize;
++	size_t paddedWidth;
+ 
+ 	paddedWidth = BitmapBytePad(width);
++
++	if (paddedWidth > 32767 || height > 32767)
++	    return NullPixmap;
++	
+ 	datasize = height * paddedWidth * depth;
+ 	pPixmap = AllocatePixmap(pScreen, datasize);
+ 	if (!pPixmap)
+--- programs/Xserver/cfb/cfbpixmap.c.orig	Fri Apr 23 21:00:12 2004
++++ programs/Xserver/cfb/cfbpixmap.c	Tue Sep  6 17:08:01 2005
+@@ -70,10 +70,13 @@ cfbCreatePixmap (pScreen, width, height,
+     int		depth;
+ {
+     PixmapPtr pPixmap;
+-    int datasize;
+-    int paddedWidth;
++    size_t datasize;
++    size_t paddedWidth;
+ 
+     paddedWidth = PixmapBytePad(width, depth);
++
++    if (paddedWidth > 32767 || height > 32767)
++	return NullPixmap;
+     datasize = height * paddedWidth;
+     pPixmap = AllocatePixmap(pScreen, datasize);
+     if (!pPixmap)
+--- programs/Xserver/dix/dispatch.c.orig	Mon Dec 13 02:23:05 2004
++++ programs/Xserver/dix/dispatch.c	Tue Sep  6 17:08:01 2005
+@@ -1506,6 +1506,23 @@ ProcCreatePixmap(client)
+ 	client->errorValue = 0;
+         return BadValue;
+     }
++    if (stuff->width > 32767 || stuff->height > 32767)
++    {
++	/* It is allowed to try and allocate a pixmap which is larger than
++	 * 32767 in either dimension. However, all of the framebuffer code
++	 * is buggy and does not reliably draw to such big pixmaps, basically
++	 * because the Region data structure operates with signed shorts
++	 * for the rectangles in it.
++	 *
++	 * Furthermore, several places in the X server computes the
++	 * size in bytes of the pixmap and tries to store it in an
++	 * integer. This integer can overflow and cause the allocated size
++	 * to be much smaller.
++	 *
++	 * So, such big pixmaps are rejected here with a BadAlloc
++	 */
++	return BadAlloc;
++    }
+     if (stuff->depth != 1)
+     {
+         pDepth = pDraw->pScreen->allowedDepths;
+--- programs/Xserver/fb/fbpixmap.c.orig	Mon Aug  9 05:40:50 2004
++++ programs/Xserver/fb/fbpixmap.c	Tue Sep  6 17:08:01 2005
+@@ -32,12 +32,14 @@ PixmapPtr
+ fbCreatePixmapBpp (ScreenPtr pScreen, int width, int height, int depth, int bpp)
+ {
+     PixmapPtr	pPixmap;
+-    int		datasize;
+-    int		paddedWidth;
++    size_t	datasize;
++    size_t	paddedWidth;
+     int		adjust;
+     int		base;
+ 
+     paddedWidth = ((width * bpp + FB_MASK) >> FB_SHIFT) * sizeof (FbBits);
++    if (paddedWidth > 32767 || height > 32767)
++	return NullPixmap;
+     datasize = height * paddedWidth;
+ #ifdef PIXPRIV
+     base = pScreen->totalPixmapSize;
+--- programs/Xserver/hw/xfree86/xaa/xaaInit.c.orig	Fri Jul 30 22:30:56 2004
++++ programs/Xserver/hw/xfree86/xaa/xaaInit.c	Tue Sep  6 17:08:01 2005
+@@ -498,6 +498,9 @@ XAACreatePixmap(ScreenPtr pScreen, int w
+     XAAPixmapPtr pPriv;
+     PixmapPtr pPix = NULL;
+     int size = w * h;
++
++    if (w > 32767 || h > 32767)
++	return NullPixmap;
+     
+     if (!infoRec->offscreenDepthsInitialized)
+ 	XAAInitializeOffscreenDepths (pScreen);
+--- programs/Xserver/hw/xfree86/xf4bpp/ppcPixmap.c.orig	Fri Apr 23 21:54:17 2004
++++ programs/Xserver/hw/xfree86/xf4bpp/ppcPixmap.c	Tue Sep  6 17:08:01 2005
+@@ -85,14 +85,18 @@ xf4bppCreatePixmap( pScreen, width, heig
+     int		depth ;
+ {
+     register PixmapPtr pPixmap  = (PixmapPtr)NULL;
+-    int size ;
++    size_t size ;
+     
+     TRACE(("xf4bppCreatePixmap(pScreen=0x%x, width=%d, height=%d, depth=%d)\n", pScreen, width, height, depth)) ;
+ 
+     if ( depth > 8 )
+ 	return (PixmapPtr) NULL ;
+ 
++    if (width > 32767 || height > 32767)
++	return (PixmapPtr) NULL ;
++    
+     size = PixmapBytePad(width, depth);
++
+     pPixmap = AllocatePixmap (pScreen, (height * size));
+     
+     if ( !pPixmap )
+--- programs/Xserver/ilbm/ilbmpixmap.c.orig	Fri Apr 23 21:54:22 2004
++++ programs/Xserver/ilbm/ilbmpixmap.c	Tue Sep  6 17:08:01 2005
+@@ -75,10 +75,12 @@ ilbmCreatePixmap(pScreen, width, height,
+ 	int				depth;
+ {
+ 	PixmapPtr pPixmap;
+-	int datasize;
+-	int paddedWidth;
++	size_t datasize;
++	size_t paddedWidth;
+ 
+ 	paddedWidth = BitmapBytePad(width);
++	if (paddedWidth > 32767 || height > 32767)
++		return NullPixmap;
+ 	datasize = height * paddedWidth * depth;
+ 	pPixmap = AllocatePixmap(pScreen, datasize);
+ 	if (!pPixmap)
+--- programs/Xserver/iplan2p4/iplpixmap.c.orig	Fri Apr 23 21:54:24 2004
++++ programs/Xserver/iplan2p4/iplpixmap.c	Tue Sep  6 17:08:01 2005
+@@ -74,12 +74,14 @@ iplCreatePixmap (pScreen, width, height,
+     int		depth;
+ {
+     PixmapPtr pPixmap;
+-    int datasize;
+-    int paddedWidth;
++    size_t datasize;
++    size_t paddedWidth;
+     int ipad=INTER_PLANES*2 - 1;
+ 
+     paddedWidth = PixmapBytePad(width, depth);
+     paddedWidth = (paddedWidth + ipad) & ~ipad;
++    if (paddedWidth > 32767 || height > 32767)
++	return NullPixmap;
+     datasize = height * paddedWidth;
+     pPixmap = AllocatePixmap(pScreen, datasize);
+     if (!pPixmap)
+--- programs/Xserver/mfb/mfbpixmap.c.orig	Fri Nov 14 17:48:57 2003
++++ programs/Xserver/mfb/mfbpixmap.c	Tue Sep  6 17:08:01 2005
+@@ -72,10 +72,12 @@ mfbCreatePixmap (pScreen, width, height,
+     int		depth;
+ {
+     PixmapPtr pPixmap;
+-    int datasize;
+-    int paddedWidth;
++    size_t datasize;
++    size_t paddedWidth;
+ 
+     if (depth != 1)
++	return NullPixmap;
++    if (width > 32767 || height > 32767)
+ 	return NullPixmap;
+     paddedWidth = BitmapBytePad(width);
+     datasize = height * paddedWidth;