Subject: glx: Integer overflow protection for non-generated render requests (v3)
References: bnc#907268, CVE-2014-8093
Patch-Mainline: Upstream
Signed-off-by: Michal Srb <msrb@suse.com>

v2:
Fix constants in __glXMap2fReqSize (Michal Srb)
Validate w/h/d for proxy targets too (Keith Packard)

v3:
Fix Map[12]Size to correctly reject order == 0 (Julien Cristau)

Reviewed-by: Keith Packard <keithp@keithp.com>
Reviewed-by: Michal Srb <msrb@suse.com>
Reviewed-by: Andy Ritger <aritger@nvidia.com>
Signed-off-by: Adam Jackson <ajax@redhat.com>
Signed-off-by: Alan Coopersmith <alan.coopersmith@oracle.com>
---
 glx/rensize.c |   77 ++++++++++++++++++++++++++++++---------------------------
 1 file changed, 41 insertions(+), 36 deletions(-)

Index: xorg-server-1.6.5/glx/rensize.c
===================================================================
--- xorg-server-1.6.5.orig/glx/rensize.c
+++ xorg-server-1.6.5/glx/rensize.c
@@ -43,16 +43,10 @@
   (((a & 0xff000000U)>>24) | ((a & 0xff0000U)>>8) | \
    ((a & 0xff00U)<<8) | ((a & 0xffU)<<24))
 
-static int Map1Size( GLint k, GLint order)
-{
-    if (order <= 0 || k < 0) return -1;
-    return k * order;
-}
-
 int __glXMap1dReqSize( const GLbyte *pc, Bool swap )
 {
     GLenum target;
-    GLint order, k;
+    GLint order;
 
     target = *(GLenum*) (pc + 16);
     order = *(GLint*) (pc + 20);
@@ -60,14 +54,15 @@ int __glXMap1dReqSize( const GLbyte *pc,
 	target = SWAPL( target );
 	order = SWAPL( order );
     }
-    k = __glMap1d_size( target );
-    return 8 * Map1Size( k, order );
+    if (order < 1)
+	return -1;
+    return safe_mul(8, safe_mul(__glMap1d_size(target), order));
 }
 
 int __glXMap1fReqSize( const GLbyte *pc, Bool swap )
 {
     GLenum target;
-    GLint order, k;
+    GLint order;
 
     target = *(GLenum *)(pc + 0);
     order = *(GLint *)(pc + 12);
@@ -75,20 +70,22 @@ int __glXMap1fReqSize( const GLbyte *pc,
 	target = SWAPL( target );
 	order = SWAPL( order );
     }
-    k = __glMap1f_size(target);
-    return 4 * Map1Size(k, order);
+    if (order < 1)
+	return -1;
+    return safe_mul(4, safe_mul(__glMap1f_size(target), order));
 }
 
 static int Map2Size(int k, int majorOrder, int minorOrder)
 {
-    if (majorOrder <= 0 || minorOrder <= 0 || k < 0) return -1;
-    return k * majorOrder * minorOrder;
+    if (majorOrder < 1 || minorOrder < 1)
+        return -1;
+    return safe_mul(k, safe_mul(majorOrder, minorOrder));
 }
 
 int __glXMap2dReqSize( const GLbyte *pc, Bool swap )
 {
     GLenum target;
-    GLint uorder, vorder, k;
+    GLint uorder, vorder;
 
     target = *(GLenum *)(pc + 32);
     uorder = *(GLint *)(pc + 36);
@@ -98,14 +95,13 @@ int __glXMap2dReqSize( const GLbyte *pc,
 	uorder = SWAPL( uorder );
 	vorder = SWAPL( vorder );
     }
-    k = __glMap2d_size( target );
-    return 8 * Map2Size( k, uorder, vorder );
+    return safe_mul(8, Map2Size(__glMap2d_size(target), uorder, vorder));
 }
 
 int __glXMap2fReqSize( const GLbyte *pc, Bool swap )
 {
     GLenum target;
-    GLint uorder, vorder, k;
+    GLint uorder, vorder;
 
     target = *(GLenum *)(pc + 0);
     uorder = *(GLint *)(pc + 12);
@@ -115,8 +111,7 @@ int __glXMap2fReqSize( const GLbyte *pc,
 	uorder = SWAPL( uorder );
 	vorder = SWAPL( vorder );
     }
-    k = __glMap2f_size( target );
-    return 4 * Map2Size( k, uorder, vorder );
+    return safe_mul(4, Map2Size(__glMap2f_size(target), uorder, vorder));
 }
 
 /**
@@ -165,14 +160,17 @@ int __glXImageSize( GLenum format, GLenu
 {
     GLint bytesPerElement, elementsPerGroup, groupsPerRow;
     GLint groupSize, rowSize, padding, imageSize;
+    
+    if (w == 0 || h == 0 || d == 0)
+        return 0;
 
     if (w < 0 || h < 0 || d < 0 ||
 	(type == GL_BITMAP &&
 	 (format != GL_COLOR_INDEX && format != GL_STENCIL_INDEX))) {
 	return -1;
     }
-    if (w==0 || h==0 || d == 0) return 0;
 
+    /* proxy targets have no data */
     switch( target ) {
     case GL_PROXY_TEXTURE_1D:
     case GL_PROXY_TEXTURE_2D:
@@ -189,6 +187,12 @@ int __glXImageSize( GLenum format, GLenu
 	return 0;
     }
 
+    /* real data has to have real sizes */
+    if (imageHeight < 0 || rowLength < 0 || skipImages < 0 || skipRows < 0)
+        return -1;
+    if (alignment != 1 && alignment != 2 && alignment != 4 && alignment != 8)
+        return -1;
+
     if (type == GL_BITMAP) {
 	if (rowLength > 0) {
 	    groupsPerRow = rowLength;
@@ -196,11 +200,14 @@ int __glXImageSize( GLenum format, GLenu
 	    groupsPerRow = w;
 	}
 	rowSize = (groupsPerRow + 7) >> 3;
+        if (rowSize < 0)
+            return -1;
 	padding = (rowSize % alignment);
 	if (padding) {
 	    rowSize += alignment - padding;
 	}
-	return ((h + skipRows) * rowSize);
+
+	return safe_mul(safe_add(h, skipRows), rowSize);
     } else {
 	switch(format) {
 	  case GL_COLOR_INDEX:
@@ -291,23 +298,27 @@ int __glXImageSize( GLenum format, GLenu
 	  default:
 	    return -1;
 	}
+	/* known safe by the switches above, not checked */
 	groupSize = bytesPerElement * elementsPerGroup;
 	if (rowLength > 0) {
 	    groupsPerRow = rowLength;
 	} else {
 	    groupsPerRow = w;
 	}
-	rowSize = groupsPerRow * groupSize;
-	padding = (rowSize % alignment);
+
+	if ((rowSize = safe_mul(groupsPerRow, groupSize)) < 0)
+	    return -1;	padding = (rowSize % alignment);
 	if (padding) {
 	    rowSize += alignment - padding;
 	}
-	if (imageHeight > 0) {
-	    imageSize = (imageHeight + skipRows) * rowSize;
-	} else {
-	    imageSize = (h + skipRows) * rowSize;
-	}
-	return ((d + skipImages) * imageSize);
+
+	if (imageHeight > 0)
+	    h = imageHeight;
+	h = safe_add(h, skipRows);
+
+	imageSize = safe_mul(h, rowSize);
+
+	return safe_mul(safe_add(d, skipImages), imageSize);
     }
 }
 
@@ -420,9 +431,8 @@ int __glXSeparableFilter2DReqSize( const
     /* XXX Should rowLength be used for either or both image? */
     image1size = __glXImageSize( format, type, 0, w, 1, 1,
 				 0, rowLength, 0, 0, alignment );
-    image1size = __GLX_PAD(image1size);
     image2size = __glXImageSize( format, type, 0, h, 1, 1,
 				 0, rowLength, 0, 0, alignment );
-    return image1size + image2size;
 
+    return safe_add(safe_pad(image1size), image2size);
 }
