From 406da0f6e211a74149ee7b6e953af4b4f423e4fb Mon Sep 17 00:00:00 2001
From: Marek Kasik <mkasik@redhat.com>
Date: Tue, 30 Apr 2019 18:47:44 +0200
Subject: [PATCH] PSOutputDev: Don't read outside of image buffer

Check whether input image is RGB or BGR to not treat
it as CMYK in those cases in PSOutputDev::checkPageSlice().

Fixes #751
---
 poppler/PSOutputDev.cc | 248 ++++++++++++++++++++++++++++++++---------
 1 file changed, 196 insertions(+), 52 deletions(-)

Index: poppler-0.24.4/poppler/PSOutputDev.cc
===================================================================
--- poppler-0.24.4.orig/poppler/PSOutputDev.cc
+++ poppler-0.24.4/poppler/PSOutputDev.cc
@@ -3225,6 +3225,15 @@ GBool PSOutputDev::checkPageSlice(Page *
       break;
     case psLevel1Sep:
       useBinary = globalParams->getPSBinary();
+
+      GfxColor inputColor;
+      GfxCMYK cmyk;
+      unsigned char cmykColor[4];
+      GfxDeviceRGBColorSpace *rgbCS;
+      SplashColorMode colorMode;
+
+      colorMode = bitmap->getMode();
+
       p = bitmap->getDataPtr();
       // Check for an all gray image
       isGray = gTrue;
@@ -3295,65 +3304,201 @@ GBool PSOutputDev::checkPageSlice(Page *
         }
       } else if (((psProcessCyan | psProcessMagenta | psProcessYellow | psProcessBlack) & ~processColors) != 0) {
 	// Color image, need to check color flags for each dot
-        for (y = 0; y < h; ++y) {
-          for (comp = 0; comp < 4; ++comp) {
-	    if (useBinary) {
-	      // Binary color image
-	      for (x = 0; x < w; ++x) {
-	        col[comp] |= p[4*x + comp];
-	        hexBuf[i++] = p[4*x + comp];
-	        if (i >= 64) {
-	          writePSBuf(hexBuf, i);
-	          i = 0;
+        switch (colorMode) {
+          case splashModeRGB8:
+          case splashModeBGR8:
+            rgbCS = new GfxDeviceRGBColorSpace();
+            for (y = 0; y < h; ++y) {
+              for (comp = 0; comp < 4; ++comp) {
+	        if (useBinary) {
+	          // Binary color image
+	          for (x = 0; x < w; ++x) {
+	            if (likely(colorMode == splashModeRGB8)) {
+                      inputColor.c[0] = byteToCol(p[3*x + 0]);
+                      inputColor.c[1] = byteToCol(p[3*x + 1]);
+                      inputColor.c[2] = byteToCol(p[3*x + 2]);
+                    } else {
+                      inputColor.c[0] = byteToCol(p[3*x + 2]);
+                      inputColor.c[1] = byteToCol(p[3*x + 1]);
+                      inputColor.c[2] = byteToCol(p[3*x + 0]);
+                    }
+                    rgbCS->getCMYK(&inputColor, &cmyk);
+                    cmykColor[0] = colToByte(cmyk.c);
+                    cmykColor[1] = colToByte(cmyk.m);
+                    cmykColor[2] = colToByte(cmyk.y);
+                    cmykColor[3] = colToByte(cmyk.k);
+
+	            col[comp] |= cmykColor[comp];
+	            hexBuf[i++] = cmykColor[comp];
+	            if (i >= 64) {
+	              writePSBuf(hexBuf, i);
+	              i = 0;
+	            }
+	          }
+	        } else {
+	          // Gray color image
+	          for (x = 0; x < w; ++x) {
+	            if (likely(colorMode == splashModeRGB8)) {
+                      inputColor.c[0] = byteToCol(p[3*x + 0]);
+                      inputColor.c[1] = byteToCol(p[3*x + 1]);
+                      inputColor.c[2] = byteToCol(p[3*x + 2]);
+                    } else {
+                      inputColor.c[0] = byteToCol(p[3*x + 2]);
+                      inputColor.c[1] = byteToCol(p[3*x + 1]);
+                      inputColor.c[2] = byteToCol(p[3*x + 0]);
+                    }
+                    rgbCS->getCMYK(&inputColor, &cmyk);
+                    cmykColor[0] = colToByte(cmyk.c);
+                    cmykColor[1] = colToByte(cmyk.m);
+                    cmykColor[2] = colToByte(cmyk.y);
+                    cmykColor[3] = colToByte(cmyk.k);
+
+	            col[comp] |= cmykColor[comp];
+	            digit = cmykColor[comp] / 16;
+	            hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
+	            digit = cmykColor[comp] % 16;
+	            hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
+	            if (i >= 64) {
+	              hexBuf[i++] = '\n';
+	              writePSBuf(hexBuf, i);
+	              i = 0;
+	            }
+	          }
 	        }
-	      }
-	    } else {
-	      // Gray color image
-	      for (x = 0; x < w; ++x) {
-	        col[comp] |= p[4*x + comp];
-	        digit = p[4*x + comp] / 16;
-	        hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
-	        digit = p[4*x + comp] % 16;
-	        hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
-	        if (i >= 64) {
-	          hexBuf[i++] = '\n';
-	          writePSBuf(hexBuf, i);
-	          i = 0;
+              }
+              p -= bitmap->getRowSize();
+            }
+            delete rgbCS;
+            break;
+          default:
+            for (y = 0; y < h; ++y) {
+              for (comp = 0; comp < 4; ++comp) {
+	        if (useBinary) {
+	          // Binary color image
+	          for (x = 0; x < w; ++x) {
+	            col[comp] |= p[4*x + comp];
+	            hexBuf[i++] = p[4*x + comp];
+	            if (i >= 64) {
+	              writePSBuf(hexBuf, i);
+	              i = 0;
+	            }
+	          }
+	        } else {
+	          // Gray color image
+	          for (x = 0; x < w; ++x) {
+	            col[comp] |= p[4*x + comp];
+	            digit = p[4*x + comp] / 16;
+	            hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
+	            digit = p[4*x + comp] % 16;
+	            hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
+	            if (i >= 64) {
+	              hexBuf[i++] = '\n';
+	              writePSBuf(hexBuf, i);
+	              i = 0;
+	            }
+	          }
 	        }
-	      }
-	    }
-          }
-          p -= bitmap->getRowSize();
+              }
+              p -= bitmap->getRowSize();
+            }
+          break;
         }
       } else {
 	// Color image, do not need to check color flags
-        for (y = 0; y < h; ++y) {
-          for (comp = 0; comp < 4; ++comp) {
-	    if (useBinary) {
-	      // Binary color image
-	      for (x = 0; x < w; ++x) {
-	        hexBuf[i++] = p[4*x + comp];
-	        if (i >= 64) {
-	          writePSBuf(hexBuf, i);
-	          i = 0;
+        switch (colorMode) {
+          case splashModeRGB8:
+          case splashModeBGR8:
+            rgbCS = new GfxDeviceRGBColorSpace();
+            for (y = 0; y < h; ++y) {
+              for (comp = 0; comp < 4; ++comp) {
+	        if (useBinary) {
+	          // Binary color image
+	          for (x = 0; x < w; ++x) {
+	            if (likely(colorMode == splashModeRGB8)) {
+                      inputColor.c[0] = byteToCol(p[3*x + 0]);
+                      inputColor.c[1] = byteToCol(p[3*x + 1]);
+                      inputColor.c[2] = byteToCol(p[3*x + 2]);
+                    } else {
+                      inputColor.c[0] = byteToCol(p[3*x + 2]);
+                      inputColor.c[1] = byteToCol(p[3*x + 1]);
+                      inputColor.c[2] = byteToCol(p[3*x + 0]);
+                    }
+                    rgbCS->getCMYK(&inputColor, &cmyk);
+                    cmykColor[0] = colToByte(cmyk.c);
+                    cmykColor[1] = colToByte(cmyk.m);
+                    cmykColor[2] = colToByte(cmyk.y);
+                    cmykColor[3] = colToByte(cmyk.k);
+
+	            hexBuf[i++] = cmykColor[comp];
+	            if (i >= 64) {
+	              writePSBuf(hexBuf, i);
+	              i = 0;
+	            }
+	          }
+	        } else {
+	          // Hex color image
+	          for (x = 0; x < w; ++x) {
+	            if (likely(colorMode == splashModeRGB8)) {
+                      inputColor.c[0] = byteToCol(p[3*x + 0]);
+                      inputColor.c[1] = byteToCol(p[3*x + 1]);
+                      inputColor.c[2] = byteToCol(p[3*x + 2]);
+                    } else {
+                      inputColor.c[0] = byteToCol(p[3*x + 2]);
+                      inputColor.c[1] = byteToCol(p[3*x + 1]);
+                      inputColor.c[2] = byteToCol(p[3*x + 0]);
+                    }
+                    rgbCS->getCMYK(&inputColor, &cmyk);
+                    cmykColor[0] = colToByte(cmyk.c);
+                    cmykColor[1] = colToByte(cmyk.m);
+                    cmykColor[2] = colToByte(cmyk.y);
+                    cmykColor[3] = colToByte(cmyk.k);
+
+	            digit = cmykColor[comp] / 16;
+	            hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
+	            digit = cmykColor[comp] % 16;
+	            hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
+	            if (i >= 64) {
+	              hexBuf[i++] = '\n';
+	              writePSBuf(hexBuf, i);
+	              i = 0;
+	            }
+	          }
 	        }
-	      }
-	    } else {
-	      // Hex color image
-	      for (x = 0; x < w; ++x) {
-	        digit = p[4*x + comp] / 16;
-	        hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
-	        digit = p[4*x + comp] % 16;
-	        hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
-	        if (i >= 64) {
-	          hexBuf[i++] = '\n';
-	          writePSBuf(hexBuf, i);
-	          i = 0;
+              }
+              p -= bitmap->getRowSize();
+            }
+            delete rgbCS;
+            break;
+          default:
+            for (y = 0; y < h; ++y) {
+              for (comp = 0; comp < 4; ++comp) {
+	        if (useBinary) {
+	          // Binary color image
+	          for (x = 0; x < w; ++x) {
+	            hexBuf[i++] = p[4*x + comp];
+	            if (i >= 64) {
+	              writePSBuf(hexBuf, i);
+	              i = 0;
+	            }
+	          }
+	        } else {
+	          // Hex color image
+	          for (x = 0; x < w; ++x) {
+	            digit = p[4*x + comp] / 16;
+	            hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
+	            digit = p[4*x + comp] % 16;
+	            hexBuf[i++] = digit + ((digit >= 10)? 'a' - 10: '0');
+	            if (i >= 64) {
+	              hexBuf[i++] = '\n';
+	              writePSBuf(hexBuf, i);
+	              i = 0;
+	            }
+	          }
 	        }
-	      }
-	    }
-          }
-          p -= bitmap->getRowSize();
+              }
+              p -= bitmap->getRowSize();
+            }
+          break;
         }
       }
       if (i != 0) {
