Backport of:

From b4cd77b1236a743dd5d94bc210534856a12e6efe Mon Sep 17 00:00:00 2001
From: mark <mark@138bc75d-0d04-0410-961f-82ee72b054a4>
Date: Fri, 21 Apr 2017 09:02:03 +0000
Subject: [PATCH] libiberty: Limit demangler maximum d_print_comp recursion
 call depth.

The fix for PR demangler/70909 and 67264 (endless demangler recursion)
catches when a demangle_component is printed in a cycle. But that doesn't
protect the call stack blowing up from non-cyclic nested types printed
recursively through d_print_comp. This can happen by a (very) long mangled
string that simply creates a very deep pointer or qualifier chain. Limit
the recursive d_print_comp call depth for a d_print_info to 1K nested
types.

libiberty/ChangeLog:

        * cp-demangle.c (MAX_RECURSION_COUNT): New constant.
        (struct d_print_info): Add recursion field.
        (d_print_init): Initialize recursion.
        (d_print_comp): Check and update d_print_info recursion depth.

git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@247056 138bc75d-0d04-0410-961f-82ee72b054a4
---
 libiberty/ChangeLog     |  7 +++++++
 libiberty/cp-demangle.c | 15 ++++++++++++---
 2 files changed, 19 insertions(+), 3 deletions(-)

#diff --git a/libiberty/ChangeLog b/libiberty/ChangeLog
#index 673eb26..34e585e 100644
#--- a/libiberty/ChangeLog
#+++ b/libiberty/ChangeLog
#@@ -1,5 +1,12 @@
# 2017-04-21  Mark Wielaard  <mark@klomp.org>
# 
#+	* cp-demangle.c (MAX_RECURSION_COUNT): New constant.
#+	(struct d_print_info): Add recursion field.
#+	(d_print_init): Initialize recursion.
#+	(d_print_comp): Check and update d_print_info recursion depth.
#+
#+2017-04-21  Mark Wielaard  <mark@klomp.org>
#+
# 	* cp-demangle.c (d_substitution): Return NULL if d_add_substitution
# 	fails.
# 
Index: gdb-7.11.1/libiberty/cp-demangle.c
===================================================================
--- gdb-7.11.1.orig/libiberty/cp-demangle.c	2017-06-08 07:54:58.210740139 -0400
+++ gdb-7.11.1/libiberty/cp-demangle.c	2017-06-08 07:54:58.206740089 -0400
@@ -321,6 +321,9 @@ struct d_info_checkpoint
   int expansion;
 };
 
+/* Maximum number of times d_print_comp may be called recursively.  */
+#define MAX_RECURSION_COUNT 1024
+
 enum { D_PRINT_BUFFER_LENGTH = 256 };
 struct d_print_info
 {
@@ -343,6 +346,9 @@ struct d_print_info
   struct d_print_mod *modifiers;
   /* Set to 1 if we saw a demangling error.  */
   int demangle_failure;
+  /* Number of times d_print_comp was recursively called.  Should not
+     be bigger than MAX_RECURSION_COUNT.  */
+  int recursion;
   /* The current index into any template argument packs we are using
      for printing, or -1 to print the whole pack.  */
   int pack_index;
@@ -4024,6 +4030,7 @@ d_print_init (struct d_print_info *dpi,
   dpi->opaque = opaque;
 
   dpi->demangle_failure = 0;
+  dpi->recursion = 0;
 
   dpi->component_stack = NULL;
 
@@ -5444,13 +5451,14 @@ d_print_comp (struct d_print_info *dpi,
 	      struct demangle_component *dc)
 {
   struct d_component_stack self;
-  if (dc == NULL || dc->d_printing > 1)
+  if (dc == NULL || dc->d_printing > 1 || dpi->recursion > MAX_RECURSION_COUNT)
     {
       d_print_error (dpi);
       return;
     }
-  else
-    dc->d_printing++;
+
+  dc->d_printing++;
+  dpi->recursion++;
 
   self.dc = dc;
   self.parent = dpi->component_stack;
@@ -5460,6 +5468,7 @@ d_print_comp (struct d_print_info *dpi,
 
   dpi->component_stack = self.parent;
   dc->d_printing--;
+  dpi->recursion--;
 }
 
 /* Print a Java dentifier.  For Java we try to handle encoded extended