commit c01f7d05b8bf37b646c87170e54d70e629c9e6ac
Author: Ulrich Drepper <drepper@redhat.com>
Date:   Mon Aug 9 21:09:37 2010 -0700

    Avoid too much stack use in fnmatch.
    
    (cherry picked from commit f15ce4d8dc139523fe0c273580b604b2453acba6)

Index: glibc-2.11.1/posix/fnmatch.c
===================================================================
--- glibc-2.11.1.orig/posix/fnmatch.c
+++ glibc-2.11.1/posix/fnmatch.c
@@ -41,6 +41,12 @@
 # include <stdlib.h>
 #endif
 
+#ifdef _LIBC
+# include <alloca.h>
+#else
+# define alloca_account(size., var) alloca (size)
+#endif
+
 /* For platform which support the ISO C amendement 1 functionality we
    support user defined character classes.  */
 #if defined _LIBC || (defined HAVE_WCTYPE_H && defined HAVE_WCHAR_H)
@@ -331,8 +337,11 @@ fnmatch (pattern, string, flags)
       mbstate_t ps;
       size_t n;
       const char *p;
+      wchar_t *wpattern_malloc = NULL;
       wchar_t *wpattern;
+      wchar_t *wstring_malloc = NULL;
       wchar_t *wstring;
+      size_t alloca_used = 0;
 
       /* Convert the strings into wide characters.  */
       memset (&ps, '\0', sizeof (ps));
@@ -344,7 +353,8 @@ fnmatch (pattern, string, flags)
 #endif
       if (__builtin_expect (n < 1024, 1))
 	{
-	  wpattern = (wchar_t *) alloca ((n + 1) * sizeof (wchar_t));
+	  wpattern = (wchar_t *) alloca_account ((n + 1) * sizeof (wchar_t),
+						 alloca_used);
 	  n = mbsrtowcs (wpattern, &p, n + 1, &ps);
 	  if (__builtin_expect (n == (size_t) -1, 0))
 	    /* Something wrong.
@@ -366,8 +376,11 @@ fnmatch (pattern, string, flags)
 	       XXX Do we have to set `errno' to something which mbsrtows hasn't
 	       already done?  */
 	    return -1;
-	  wpattern = (wchar_t *) alloca ((n + 1) * sizeof (wchar_t));
+	  wpattern_malloc = wpattern
+	    = (wchar_t *) malloc ((n + 1) * sizeof (wchar_t));
 	  assert (mbsinit (&ps));
+	  if (wpattern == NULL)
+	    return -2;
 	  (void) mbsrtowcs (wpattern, &pattern, n + 1, &ps);
 	}
 
@@ -380,11 +393,16 @@ fnmatch (pattern, string, flags)
       p = string;
       if (__builtin_expect (n < 1024, 1))
 	{
-	  wstring = (wchar_t *) alloca ((n + 1) * sizeof (wchar_t));
+	  wstring = (wchar_t *) alloca_account ((n + 1) * sizeof (wchar_t),
+						alloca_used);
 	  n = mbsrtowcs (wstring, &p, n + 1, &ps);
 	  if (__builtin_expect (n == (size_t) -1, 0))
-	    /* Something wrong.  Fall back to single byte matching.  */
-	    goto try_singlebyte;
+	    {
+	      /* Something wrong.  Fall back to single byte matching.  */
+	    free_try_singlebyte:
+	      free (wpattern_malloc);
+	      goto try_singlebyte;
+	    }
 	  if (p)
 	    {
 	      memset (&ps, '\0', sizeof (ps));
@@ -397,14 +415,27 @@ fnmatch (pattern, string, flags)
 	  n = mbsrtowcs (NULL, &string, 0, &ps);
 	  if (__builtin_expect (n == (size_t) -1, 0))
 	    /* Something wrong.  Fall back to single byte matching.  */
-	    goto try_singlebyte;
-	  wstring = (wchar_t *) alloca ((n + 1) * sizeof (wchar_t));
+	    goto free_try_singlebyte;
+
+	  wstring_malloc = wstring
+	    = (wchar_t *) malloc ((n + 1) * sizeof (wchar_t));
+	  if (wstring == NULL)
+	    {
+	      free (wpattern_malloc);
+	      return -2;
+	    }
 	  assert (mbsinit (&ps));
 	  (void) mbsrtowcs (wstring, &string, n + 1, &ps);
 	}
 
-      return internal_fnwmatch (wpattern, wstring, wstring + n,
-				flags & FNM_PERIOD, flags, NULL);
+      int res = internal_fnwmatch (wpattern, wstring, wstring + n,
+				   flags & FNM_PERIOD, flags, NULL,
+				   alloca_used);
+
+      free (wstring_malloc);
+      free (wpattern_malloc);
+
+      return res;
 
     try_singlebyte:
       pattern = orig_pattern;
@@ -412,7 +443,7 @@ fnmatch (pattern, string, flags)
 # endif  /* mbstate_t and mbsrtowcs or _LIBC.  */
 
   return internal_fnmatch (pattern, string, string + strlen (string),
-			   flags & FNM_PERIOD, flags, NULL);
+			   flags & FNM_PERIOD, flags, NULL, 0);
 }
 
 # ifdef _LIBC
Index: glibc-2.11.1/posix/fnmatch_loop.c
===================================================================
--- glibc-2.11.1.orig/posix/fnmatch_loop.c
+++ glibc-2.11.1/posix/fnmatch_loop.c
@@ -28,22 +28,24 @@ struct STRUCT
    it matches, nonzero if not.  */
 static int FCT (const CHAR *pattern, const CHAR *string,
 		const CHAR *string_end, int no_leading_period, int flags,
-		struct STRUCT *ends)
+		struct STRUCT *ends, size_t alloca_used)
      internal_function;
 static int EXT (INT opt, const CHAR *pattern, const CHAR *string,
-		const CHAR *string_end, int no_leading_period, int flags)
+		const CHAR *string_end, int no_leading_period, int flags,
+		size_t alloca_used)
      internal_function;
 static const CHAR *END (const CHAR *patternp) internal_function;
 
 static int
 internal_function
-FCT (pattern, string, string_end, no_leading_period, flags, ends)
+FCT (pattern, string, string_end, no_leading_period, flags, ends, alloca_used)
      const CHAR *pattern;
      const CHAR *string;
      const CHAR *string_end;
      int no_leading_period;
      int flags;
      struct STRUCT *ends;
+     size_t alloca_used;
 {
   register const CHAR *p = pattern, *n = string;
   register UCHAR c;
@@ -67,10 +69,8 @@ FCT (pattern, string, string_end, no_lea
 	case L('?'):
 	  if (__builtin_expect (flags & FNM_EXTMATCH, 0) && *p == '(')
 	    {
-	      int res;
-
-	      res = EXT (c, p, n, string_end, no_leading_period,
-			 flags);
+	      int res = EXT (c, p, n, string_end, no_leading_period,
+			     flags, alloca_used);
 	      if (res != -1)
 		return res;
 	    }
@@ -99,10 +99,8 @@ FCT (pattern, string, string_end, no_lea
 	case L('*'):
 	  if (__builtin_expect (flags & FNM_EXTMATCH, 0) && *p == '(')
 	    {
-	      int res;
-
-	      res = EXT (c, p, n, string_end, no_leading_period,
-			 flags);
+	      int res = EXT (c, p, n, string_end, no_leading_period,
+			     flags, alloca_used);
 	      if (res != -1)
 		return res;
 	    }
@@ -191,7 +189,7 @@ FCT (pattern, string, string_end, no_lea
 
 		  for (--p; n < endp; ++n, no_leading_period = 0)
 		    if (FCT (p, n, string_end, no_leading_period, flags2,
-			     &end) == 0)
+			     &end, alloca_used) == 0)
 		      goto found;
 		}
 	      else if (c == L('/') && (flags & FNM_FILE_NAME))
@@ -200,7 +198,7 @@ FCT (pattern, string, string_end, no_lea
 		    ++n;
 		  if (n < string_end && *n == L('/')
 		      && (FCT (p, n + 1, string_end, flags & FNM_PERIOD, flags,
-			       NULL) == 0))
+			       NULL, alloca_used) == 0))
 		    return 0;
 		}
 	      else
@@ -214,7 +212,7 @@ FCT (pattern, string, string_end, no_lea
 		  for (--p; n < endp; ++n, no_leading_period = 0)
 		    if (FOLD ((UCHAR) *n) == c
 			&& (FCT (p, n, string_end, no_leading_period, flags2,
-				 &end) == 0))
+				 &end, alloca_used) == 0))
 		      {
 		      found:
 			if (end.pattern == NULL)
@@ -749,7 +747,7 @@ FCT (pattern, string, string_end, no_lea
 					       _NL_COLLATE_SYMB_EXTRAMB);
 
 				/* Locate the character in the hashing
-                                   table.  */
+				   table.  */
 				hash = elem_hash (str, c1);
 
 				idx = 0;
@@ -971,9 +969,8 @@ FCT (pattern, string, string_end, no_lea
 	case L('!'):
 	  if (__builtin_expect (flags & FNM_EXTMATCH, 0) && *p == '(')
 	    {
-	      int res;
-
-	      res = EXT (c, p, n, string_end, no_leading_period, flags);
+	      int res = EXT (c, p, n, string_end, no_leading_period, flags,
+			     alloca_used);
 	      if (res != -1)
 		return res;
 	    }
@@ -1052,26 +1049,32 @@ END (const CHAR *pattern)
 static int
 internal_function
 EXT (INT opt, const CHAR *pattern, const CHAR *string, const CHAR *string_end,
-     int no_leading_period, int flags)
+     int no_leading_period, int flags, size_t alloca_used)
 {
   const CHAR *startp;
   int level;
   struct patternlist
   {
     struct patternlist *next;
+    CHAR malloced;
     CHAR str[0];
   } *list = NULL;
   struct patternlist **lastp = &list;
   size_t pattern_len = STRLEN (pattern);
+  int any_malloced = 0;
   const CHAR *p;
   const CHAR *rs;
+  int retval = 0;
 
   /* Parse the pattern.  Store the individual parts in the list.  */
   level = 0;
   for (startp = p = pattern + 1; level >= 0; ++p)
     if (*p == L('\0'))
-      /* This is an invalid pattern.  */
-      return -1;
+      {
+	/* This is an invalid pattern.  */
+	retval = -1;
+	goto out;
+      }
     else if (*p == L('['))
       {
 	/* Handle brackets special.  */
@@ -1088,8 +1091,11 @@ EXT (INT opt, const CHAR *pattern, const
 	/* Skip over all characters of the list.  */
 	while (*p != L(']'))
 	  if (*p++ == L('\0'))
-	    /* This is no valid pattern.  */
-	    return -1;
+	    {
+	      /* This is no valid pattern.  */
+	      retval = -1;
+	      goto out;
+	    }
       }
     else if ((*p == L('?') || *p == L('*') || *p == L('+') || *p == L('@')
 	      || *p == L('!')) && p[1] == L('('))
@@ -1102,15 +1108,25 @@ EXT (INT opt, const CHAR *pattern, const
 	    /* This means we found the end of the pattern.  */
 #define NEW_PATTERN \
 	    struct patternlist *newp;					      \
-									      \
-	    if (opt == L('?') || opt == L('@'))				      \
-	      newp = alloca (sizeof (struct patternlist)		      \
-			     + (pattern_len * sizeof (CHAR)));		      \
+	    size_t slen = (opt == L('?') || opt == L('@')		      \
+			   ? pattern_len : (p - startp + 1));		      \
+	    slen = sizeof (struct patternlist) + (slen * sizeof (CHAR));      \
+	    int malloced = ! __libc_use_alloca (alloca_used + slen);	      \
+	    if (__builtin_expect (malloced, 0))				      \
+	      {								      \
+		newp = malloc (slen);					      \
+		if (newp == NULL)					      \
+		  {							      \
+		    retval = -2;					      \
+		    goto out;						      \
+		  }							      \
+		any_malloced = 1;					      \
+	      }								      \
 	    else							      \
-	      newp = alloca (sizeof (struct patternlist)		      \
-			     + ((p - startp + 1) * sizeof (CHAR)));	      \
-	    *((CHAR *) MEMPCPY (newp->str, startp, p - startp)) = L('\0');    \
+	      newp = alloca_account (slen, alloca_used);		      \
 	    newp->next = NULL;						      \
+	    newp->malloced = malloced;					      \
+	    *((CHAR *) MEMPCPY (newp->str, startp, p - startp)) = L('\0');    \
 	    *lastp = newp;						      \
 	    lastp = &newp->next
 	    NEW_PATTERN;
@@ -1131,8 +1147,9 @@ EXT (INT opt, const CHAR *pattern, const
   switch (opt)
     {
     case L('*'):
-      if (FCT (p, string, string_end, no_leading_period, flags, NULL) == 0)
-	return 0;
+      if (FCT (p, string, string_end, no_leading_period, flags, NULL,
+	       alloca_used) == 0)
+	goto success;
       /* FALLTHROUGH */
 
     case L('+'):
@@ -1143,7 +1160,7 @@ EXT (INT opt, const CHAR *pattern, const
 	       current pattern.  */
 	    if (FCT (list->str, string, rs, no_leading_period,
 		     flags & FNM_FILE_NAME ? flags : flags & ~FNM_PERIOD,
-		     NULL) == 0
+		     NULL, alloca_used) == 0
 		/* This was successful.  Now match the rest with the rest
 		   of the pattern.  */
 		&& (FCT (p, rs, string_end,
@@ -1151,7 +1168,7 @@ EXT (INT opt, const CHAR *pattern, const
 			 ? no_leading_period
 			 : rs[-1] == '/' && NO_LEADING_PERIOD (flags) ? 1 : 0,
 			 flags & FNM_FILE_NAME
-			 ? flags : flags & ~FNM_PERIOD, NULL) == 0
+			 ? flags : flags & ~FNM_PERIOD, NULL, alloca_used) == 0
 		    /* This didn't work.  Try the whole pattern.  */
 		    || (rs != string
 			&& FCT (pattern - 1, rs, string_end,
@@ -1160,18 +1177,21 @@ EXT (INT opt, const CHAR *pattern, const
 				: (rs[-1] == '/' && NO_LEADING_PERIOD (flags)
 				   ? 1 : 0),
 				flags & FNM_FILE_NAME
-				? flags : flags & ~FNM_PERIOD, NULL) == 0)))
+				? flags : flags & ~FNM_PERIOD, NULL,
+				alloca_used) == 0)))
 	      /* It worked.  Signal success.  */
-	      return 0;
+	      goto success;
 	}
       while ((list = list->next) != NULL);
 
       /* None of the patterns lead to a match.  */
-      return FNM_NOMATCH;
+      retval = FNM_NOMATCH;
+      break;
 
     case L('?'):
-      if (FCT (p, string, string_end, no_leading_period, flags, NULL) == 0)
-	return 0;
+      if (FCT (p, string, string_end, no_leading_period, flags, NULL,
+	       alloca_used) == 0)
+	goto success;
       /* FALLTHROUGH */
 
     case L('@'):
@@ -1183,13 +1203,14 @@ EXT (INT opt, const CHAR *pattern, const
 	if (FCT (STRCAT (list->str, p), string, string_end,
 		 no_leading_period,
 		 flags & FNM_FILE_NAME ? flags : flags & ~FNM_PERIOD,
-		 NULL) == 0)
+		 NULL, alloca_used) == 0)
 	  /* It worked.  Signal success.  */
-	  return 0;
+	  goto success;
       while ((list = list->next) != NULL);
 
       /* None of the patterns lead to a match.  */
-      return FNM_NOMATCH;
+      retval = FNM_NOMATCH;
+      break;
 
     case L('!'):
       for (rs = string; rs <= string_end; ++rs)
@@ -1199,7 +1220,7 @@ EXT (INT opt, const CHAR *pattern, const
 	  for (runp = list; runp != NULL; runp = runp->next)
 	    if (FCT (runp->str, string, rs,  no_leading_period,
 		     flags & FNM_FILE_NAME ? flags : flags & ~FNM_PERIOD,
-		     NULL) == 0)
+		     NULL, alloca_used) == 0)
 	      break;
 
 	  /* If none of the patterns matched see whether the rest does.  */
@@ -1209,21 +1230,34 @@ EXT (INT opt, const CHAR *pattern, const
 		       ? no_leading_period
 		       : rs[-1] == '/' && NO_LEADING_PERIOD (flags) ? 1 : 0,
 		       flags & FNM_FILE_NAME ? flags : flags & ~FNM_PERIOD,
-		       NULL) == 0))
+		       NULL, alloca_used) == 0))
 	    /* This is successful.  */
-	    return 0;
+	    goto success;
 	}
 
       /* None of the patterns together with the rest of the pattern
 	 lead to a match.  */
-      return FNM_NOMATCH;
+      retval = FNM_NOMATCH;
+      break;
 
     default:
       assert (! "Invalid extended matching operator");
+      retval = -1;
       break;
     }
 
-  return -1;
+ success:
+ out:
+  if (any_malloced)
+    while (list != NULL)
+      {
+	struct patternlist *old = list;
+	list = list->next;
+	if (old->malloced)
+	  free (old);
+      }
+
+  return retval;
 }
 
 
Index: glibc-2.11.1/posix/Makefile
===================================================================
--- glibc-2.11.1.orig/posix/Makefile
+++ glibc-2.11.1/posix/Makefile
@@ -112,7 +112,8 @@ generated := $(addprefix wordexp-test-re
 	     tst-rxspencer-mem tst-rxspencer.mtrace tst-getconf.out \
 	     tst-pcre-mem tst-pcre.mtrace tst-boost-mem tst-boost.mtrace \
 	     bug-ga2.mtrace bug-ga2-mem bug-glob2.mtrace bug-glob2-mem \
-	     tst-vfork3-mem tst-vfork3.mtrace getconf.speclist
+	     tst-vfork3-mem tst-vfork3.mtrace getconf.speclist \
+	     tst-fnmatch-mem tst-fnmatch.mtrace
 
 include ../Rules
 
@@ -224,7 +225,7 @@ ifeq (no,$(cross-compiling))
 tests: $(objpfx)bug-regex2-mem $(objpfx)bug-regex14-mem \
   $(objpfx)bug-regex21-mem $(objpfx)tst-rxspencer-mem \
   $(objpfx)tst-pcre-mem $(objpfx)tst-boost-mem $(objpfx)tst-getconf.out \
-  $(objpfx)bug-glob2-mem $(objpfx)tst-vfork3-mem
+  $(objpfx)bug-glob2-mem $(objpfx)tst-vfork3-mem $(objpfx)tst-fnmatch-mem
 xtests: $(objpfx)bug-ga2-mem
 endif
 
@@ -236,6 +237,11 @@ annexc-CFLAGS = -O
 $(objpfx)annexc: annexc.c
 	$(native-compile)
 
+tst-fnmatch-ENV += MALLOC_TRACE=$(objpfx)tst-fnmatch.mtrace
+
+$(objpfx)tst-fnmatch-mem: $(objpfx)tst-fnmatch.out
+	$(common-objpfx)malloc/mtrace $(objpfx)tst-fnmatch.mtrace > $@
+
 bug-regex2-ENV = MALLOC_TRACE=$(objpfx)bug-regex2.mtrace
 
 $(objpfx)bug-regex2-mem: $(objpfx)bug-regex2.out
Index: glibc-2.11.1/posix/tst-fnmatch.c
===================================================================
--- glibc-2.11.1.orig/posix/tst-fnmatch.c
+++ glibc-2.11.1/posix/tst-fnmatch.c
@@ -25,6 +25,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <sys/types.h>
+#include <mcheck.h>
 
 
 static char *next_input (char **line, int first, int last);
@@ -46,6 +47,8 @@ main (void)
   size_t escpatternlen = 0;
   int nr = 0;
 
+  mtrace ();
+
   /* Read lines from stdin with the following format:
 
        locale  input-string  match-string  flags  result
