From a94def348a72786e433e19dbc7c8cdd5bef02e11 Mon Sep 17 00:00:00 2001
From: Andrew Murray <radarhere@users.noreply.github.com>
Date: Sat, 20 Jun 2026 11:53:14 +1000
Subject: [PATCH 1/2] Validate large filter sizes when initializing RankFilter

---
 Tests/test_image_filter.py | 3 +++
 src/PIL/ImageFilter.py     | 3 +++
 2 files changed, 6 insertions(+)

Index: pillow-11.3.0/Tests/test_image_filter.py
===================================================================
--- pillow-11.3.0.orig/Tests/test_image_filter.py
+++ pillow-11.3.0/Tests/test_image_filter.py
@@ -141,6 +141,12 @@ def test_rankfilter_properties() -> None
     assert rankfilter.size == 1
     assert rankfilter.rank == 2
 
+    with pytest.raises(ValueError, match="filter size too large"):
+        ImageFilter.RankFilter(23171, 1)
+    im = Image.new("1", (1, 1))
+    with pytest.raises(ValueError, match="filter size too large"):
+        im.im.expand(23171)
+
 
 def test_builtinfilter_p() -> None:
     builtin_filter = ImageFilter.BuiltinFilter()
Index: pillow-11.3.0/src/PIL/ImageFilter.py
===================================================================
--- pillow-11.3.0.orig/src/PIL/ImageFilter.py
+++ pillow-11.3.0/src/PIL/ImageFilter.py
@@ -96,6 +96,9 @@ class RankFilter(Filter):
     name = "Rank"
 
     def __init__(self, size: int, rank: int) -> None:
+        if size * size * 4 > (2**31 - 1):
+            msg = "filter size too large"
+            raise ValueError(msg)
         self.size = size
         self.rank = rank
 
@@ -103,7 +106,7 @@ class RankFilter(Filter):
         if image.mode == "P":
             msg = "cannot filter palette images"
             raise ValueError(msg)
-        image = image.expand(self.size // 2, self.size // 2)
+        image = image.expand(self.size // 2)
         return image.rankfilter(self.size, self.rank)
 
 
Index: pillow-11.3.0/src/libImaging/Filter.c
===================================================================
--- pillow-11.3.0.orig/src/libImaging/Filter.c
+++ pillow-11.3.0/src/libImaging/Filter.c
@@ -51,47 +51,50 @@ clip32(float in) {
 }
 
 Imaging
-ImagingExpand(Imaging imIn, int xmargin, int ymargin) {
+ImagingExpand(Imaging imIn, int margin) {
     Imaging imOut;
     int x, y;
     ImagingSectionCookie cookie;
 
-    if (xmargin < 0 && ymargin < 0) {
+    if (margin < 0) {
         return (Imaging)ImagingError_ValueError("bad kernel size");
     }
+    if (margin > INT_MAX / (margin * (int)sizeof(FLOAT32))) {
+        return (Imaging)ImagingError_ValueError("filter size too large");
+    }
 
     imOut = ImagingNewDirty(
-        imIn->mode, imIn->xsize + 2 * xmargin, imIn->ysize + 2 * ymargin
+        imIn->mode, imIn->xsize + 2 * margin, imIn->ysize + 2 * margin
     );
     if (!imOut) {
         return NULL;
     }
 
-#define EXPAND_LINE(type, image, yin, yout)                        \
-    {                                                              \
-        for (x = 0; x < xmargin; x++) {                            \
-            imOut->image[yout][x] = imIn->image[yin][0];           \
-        }                                                          \
-        for (x = 0; x < imIn->xsize; x++) {                        \
-            imOut->image[yout][x + xmargin] = imIn->image[yin][x]; \
-        }                                                          \
-        for (x = 0; x < xmargin; x++) {                            \
-            imOut->image[yout][xmargin + imIn->xsize + x] =        \
-                imIn->image[yin][imIn->xsize - 1];                 \
-        }                                                          \
+#define EXPAND_LINE(type, image, yin, yout)                       \
+    {                                                             \
+        for (x = 0; x < margin; x++) {                            \
+            imOut->image[yout][x] = imIn->image[yin][0];          \
+        }                                                         \
+        for (x = 0; x < imIn->xsize; x++) {                       \
+            imOut->image[yout][x + margin] = imIn->image[yin][x]; \
+        }                                                         \
+        for (x = 0; x < margin; x++) {                            \
+            imOut->image[yout][margin + imIn->xsize + x] =        \
+                imIn->image[yin][imIn->xsize - 1];                \
+        }                                                         \
     }
 
-#define EXPAND(type, image)                                                       \
-    {                                                                             \
-        for (y = 0; y < ymargin; y++) {                                           \
-            EXPAND_LINE(type, image, 0, y);                                       \
-        }                                                                         \
-        for (y = 0; y < imIn->ysize; y++) {                                       \
-            EXPAND_LINE(type, image, y, y + ymargin);                             \
-        }                                                                         \
-        for (y = 0; y < ymargin; y++) {                                           \
-            EXPAND_LINE(type, image, imIn->ysize - 1, ymargin + imIn->ysize + y); \
-        }                                                                         \
+#define EXPAND(type, image)                                                      \
+    {                                                                            \
+        for (y = 0; y < margin; y++) {                                           \
+            EXPAND_LINE(type, image, 0, y);                                      \
+        }                                                                        \
+        for (y = 0; y < imIn->ysize; y++) {                                      \
+            EXPAND_LINE(type, image, y, y + margin);                             \
+        }                                                                        \
+        for (y = 0; y < margin; y++) {                                           \
+            EXPAND_LINE(type, image, imIn->ysize - 1, margin + imIn->ysize + y); \
+        }                                                                        \
     }
 
     ImagingSectionEnter(&cookie);
Index: pillow-11.3.0/src/_imaging.c
===================================================================
--- pillow-11.3.0.orig/src/_imaging.c
+++ pillow-11.3.0/src/_imaging.c
@@ -1104,12 +1104,12 @@ _crop(ImagingObject *self, PyObject *arg
 
 static PyObject *
 _expand_image(ImagingObject *self, PyObject *args) {
-    int x, y;
-    if (!PyArg_ParseTuple(args, "ii", &x, &y)) {
+    int m;
+    if (!PyArg_ParseTuple(args, "i", &m)) {
         return NULL;
     }
 
-    return PyImagingNew(ImagingExpand(self->image, x, y));
+    return PyImagingNew(ImagingExpand(self->image, m));
 }
 
 static PyObject *
Index: pillow-11.3.0/src/libImaging/Imaging.h
===================================================================
--- pillow-11.3.0.orig/src/libImaging/Imaging.h
+++ pillow-11.3.0/src/libImaging/Imaging.h
@@ -318,7 +318,7 @@ ImagingConvertTransparent(Imaging im, co
 extern Imaging
 ImagingCrop(Imaging im, int x0, int y0, int x1, int y1);
 extern Imaging
-ImagingExpand(Imaging im, int x, int y);
+ImagingExpand(Imaging im, int m);
 extern Imaging
 ImagingFill(Imaging im, const void *ink);
 extern int
Index: pillow-11.3.0/Tests/test_image.py
===================================================================
--- pillow-11.3.0.orig/Tests/test_image.py
+++ pillow-11.3.0/Tests/test_image.py
@@ -293,33 +293,6 @@ class TestImage:
         assert item is not None
         assert item != num
 
-    def test_expand_x(self) -> None:
-        # Arrange
-        im = hopper()
-        orig_size = im.size
-        xmargin = 5
-
-        # Act
-        im = im._expand(xmargin)
-
-        # Assert
-        assert im.size[0] == orig_size[0] + 2 * xmargin
-        assert im.size[1] == orig_size[1] + 2 * xmargin
-
-    def test_expand_xy(self) -> None:
-        # Arrange
-        im = hopper()
-        orig_size = im.size
-        xmargin = 5
-        ymargin = 3
-
-        # Act
-        im = im._expand(xmargin, ymargin)
-
-        # Assert
-        assert im.size[0] == orig_size[0] + 2 * xmargin
-        assert im.size[1] == orig_size[1] + 2 * ymargin
-
     def test_getbands(self) -> None:
         # Assert
         assert hopper("RGB").getbands() == ("R", "G", "B")
