Subject: glx: Be more strict about rejecting invalid image sizes
References: bnc#907268, CVE-2014-8093
Patch-Mainline: Upstream
Signed-off-by: Michal Srb <msrb@suse.com>

Before this we'd just clamp the image size to 0, which was just
hideously stupid; if the parameters were such that they'd overflow an
integer, you'd allocate a small buffer, then pass huge values into (say)
ReadPixels, and now you're scribbling over arbitrary server memory.

Reviewed-by: Keith Packard <keithp@keithp.com>
Reviewed-by: Julien Cristau <jcristau@debian.org>
Reviewed-by: Michal Srb <msrb@suse.com>
Reviewed-by: Andy Ritger <aritger@nvidia.com>
Signed-off-by: Adam Jackson <ajax@redhat.com>
Signed-off-by: Alan Coopersmith <alan.coopersmith@oracle.com>
---
 glx/singlepix.c     |   16 ++++++++--------
 glx/singlepixswap.c |   16 ++++++++--------
 2 files changed, 16 insertions(+), 16 deletions(-)

Index: xorg-server-1.6.5/glx/singlepix.c
===================================================================
--- xorg-server-1.6.5.orig/glx/singlepix.c
+++ xorg-server-1.6.5/glx/singlepix.c
@@ -68,7 +68,7 @@ int __glXDisp_ReadPixels(__GLXclientStat
     swapBytes = *(GLboolean *)(pc + 24);
     lsbFirst = *(GLboolean *)(pc + 25);
     compsize = __glReadPixels_size(format,type,width,height);
-    if (compsize < 0) compsize = 0;
+    if (compsize < 0) return BadLength;
 
     CALL_PixelStorei( GET_DISPATCH(), (GL_PACK_SWAP_BYTES, swapBytes) );
     CALL_PixelStorei( GET_DISPATCH(), (GL_PACK_LSB_FIRST, lsbFirst) );
@@ -128,7 +128,7 @@ int __glXDisp_GetTexImage(__GLXclientSta
      * are illegal, but then width, height, and depth would still be zero anyway.
      */
     compsize = __glGetTexImage_size(target,level,format,type,width,height,depth);
-    if (compsize < 0) compsize = 0;
+    if (compsize < 0) return BadLength;
 
     CALL_PixelStorei( GET_DISPATCH(), (GL_PACK_SWAP_BYTES, swapBytes) );
     __GLX_GET_ANSWER_BUFFER(answer,cl,compsize,1);
@@ -222,8 +222,8 @@ static int GetSeparableFilter(__GLXclien
     compsize = __glGetTexImage_size(target,1,format,type,width,1,1);
     compsize2 = __glGetTexImage_size(target,1,format,type,height,1,1);
 
-    if (compsize < 0) compsize = 0;
-    if (compsize2 < 0) compsize2 = 0;
+    if (compsize < 0) return BadLength;
+    if (compsize2 < 0) return BadLength;
     compsize = __GLX_PAD(compsize);
     compsize2 = __GLX_PAD(compsize2);
 
@@ -302,7 +302,7 @@ static int GetConvolutionFilter(__GLXcli
      * are illegal, but then width and height would still be zero anyway.
      */
     compsize = __glGetTexImage_size(target,1,format,type,width,height,1);
-    if (compsize < 0) compsize = 0;
+    if (compsize < 0) return BadLength;
 
     CALL_PixelStorei(GET_DISPATCH(), (GL_PACK_SWAP_BYTES, swapBytes));
     __GLX_GET_ANSWER_BUFFER(answer,cl,compsize,1);
@@ -371,7 +371,7 @@ static int GetHistogram(__GLXclientState
      * are illegal, but then width would still be zero anyway.
      */
     compsize = __glGetTexImage_size(target,1,format,type,width,1,1);
-    if (compsize < 0) compsize = 0;
+    if (compsize < 0) return BadLength;
 
     CALL_PixelStorei(GET_DISPATCH(), (GL_PACK_SWAP_BYTES, swapBytes));
     __GLX_GET_ANSWER_BUFFER(answer,cl,compsize,1);
@@ -427,7 +427,7 @@ static int GetMinmax(__GLXclientState *c
     reset = *(GLboolean *)(pc + 13);
 
     compsize = __glGetTexImage_size(target,1,format,type,2,1,1);
-    if (compsize < 0) compsize = 0;
+    if (compsize < 0) return BadLength;
 
     CALL_PixelStorei(GET_DISPATCH(), (GL_PACK_SWAP_BYTES, swapBytes));
     __GLX_GET_ANSWER_BUFFER(answer,cl,compsize,1);
@@ -488,7 +488,7 @@ static int GetColorTable(__GLXclientStat
      * are illegal, but then width would still be zero anyway.
      */
     compsize = __glGetTexImage_size(target,1,format,type,width,1,1);
-    if (compsize < 0) compsize = 0;
+    if (compsize < 0) return BadLength;
 
     CALL_PixelStorei(GET_DISPATCH(), (GL_PACK_SWAP_BYTES, swapBytes));
     __GLX_GET_ANSWER_BUFFER(answer,cl,compsize,1);
Index: xorg-server-1.6.5/glx/singlepixswap.c
===================================================================
--- xorg-server-1.6.5.orig/glx/singlepixswap.c
+++ xorg-server-1.6.5/glx/singlepixswap.c
@@ -77,7 +77,7 @@ int __glXDispSwap_ReadPixels(__GLXclient
     swapBytes = *(GLboolean *)(pc + 24);
     lsbFirst = *(GLboolean *)(pc + 25);
     compsize = __glReadPixels_size(format,type,width,height);
-    if (compsize < 0) compsize = 0;
+    if (compsize < 0) return BadLength;
 
     CALL_PixelStorei( GET_DISPATCH(), (GL_PACK_SWAP_BYTES, !swapBytes) );
     CALL_PixelStorei( GET_DISPATCH(), (GL_PACK_LSB_FIRST, lsbFirst) );
@@ -146,7 +146,7 @@ int __glXDispSwap_GetTexImage(__GLXclien
      * are illegal, but then width, height, and depth would still be zero anyway.
      */
     compsize = __glGetTexImage_size(target,level,format,type,width,height,depth);
-    if (compsize < 0) compsize = 0;
+    if (compsize < 0) return BadLength;
 
     CALL_PixelStorei( GET_DISPATCH(), (GL_PACK_SWAP_BYTES, !swapBytes) );
     __GLX_GET_ANSWER_BUFFER(answer,cl,compsize,1);
@@ -252,8 +252,8 @@ static int GetSeparableFilter(__GLXclien
     compsize = __glGetTexImage_size(target,1,format,type,width,1,1);
     compsize2 = __glGetTexImage_size(target,1,format,type,height,1,1);
 
-    if (compsize < 0) compsize = 0;
-    if (compsize2 < 0) compsize2 = 0;
+    if (compsize < 0) return BadLength;
+    if (compsize2 < 0) return BadLength;
     compsize = __GLX_PAD(compsize);
     compsize2 = __GLX_PAD(compsize2);
 
@@ -336,7 +336,7 @@ static int GetConvolutionFilter(__GLXcli
      * are illegal, but then width and height would still be zero anyway.
      */
     compsize = __glGetTexImage_size(target,1,format,type,width,height,1);
-    if (compsize < 0) compsize = 0;
+    if (compsize < 0) return BadLength;
 
     CALL_PixelStorei( GET_DISPATCH(), (GL_PACK_SWAP_BYTES, !swapBytes) );
     __GLX_GET_ANSWER_BUFFER(answer,cl,compsize,1);
@@ -411,7 +411,7 @@ static int GetHistogram(__GLXclientState
      * are illegal, but then width would still be zero anyway.
      */
     compsize = __glGetTexImage_size(target,1,format,type,width,1,1);
-    if (compsize < 0) compsize = 0;
+    if (compsize < 0) return BadLength;
 
     CALL_PixelStorei( GET_DISPATCH(), (GL_PACK_SWAP_BYTES, !swapBytes) );
     __GLX_GET_ANSWER_BUFFER(answer,cl,compsize,1);
@@ -473,7 +473,7 @@ static int GetMinmax(__GLXclientState *c
     reset = *(GLboolean *)(pc + 13);
 
     compsize = __glGetTexImage_size(target,1,format,type,2,1,1);
-    if (compsize < 0) compsize = 0;
+    if (compsize < 0) return BadLength;
 
     CALL_PixelStorei( GET_DISPATCH(), (GL_PACK_SWAP_BYTES, !swapBytes) );
     __GLX_GET_ANSWER_BUFFER(answer,cl,compsize,1);
@@ -538,7 +538,7 @@ static int GetColorTable(__GLXclientStat
      * are illegal, but then width would still be zero anyway.
      */
     compsize = __glGetTexImage_size(target,1,format,type,width,1,1);
-    if (compsize < 0) compsize = 0;
+    if (compsize < 0) return BadLength;
 
     CALL_PixelStorei( GET_DISPATCH(), (GL_PACK_SWAP_BYTES, !swapBytes) );
     __GLX_GET_ANSWER_BUFFER(answer,cl,compsize,1);
