Backported to ver. 1.17 from the original upstream commit:

From 62c2124b820365e78f437a5c2e839fa7514412d4 Mon Sep 17 00:00:00 2001
From: Steffen Jaeckel <s@jaeckel.eu>
Date: Tue, 10 Apr 2018 04:38:58 +0200
Subject: [PATCH] implement DER resursion [sic] limit

Index: libtomcrypt-1.17/src/misc/crypt/crypt.c
===================================================================
--- libtomcrypt-1.17/src/misc/crypt/crypt.c
+++ libtomcrypt-1.17/src/misc/crypt/crypt.c	2018-07-21 21:21:29.577208069 +0200
@@ -14,6 +14,8 @@
   @file crypt.c
   Build strings, Tom St Denis
 */  
+#define NAME_VALUE(s) #s"="NAME(s)
+#define NAME(s) #s
 
 const char *crypt_build_settings =
    "LibTomCrypt " SCRYPT " (Tom St Denis, tomstdenis@gmail.com)\n"
@@ -320,6 +322,7 @@
 #endif
 #if defined(LTC_DER)
     " LTC_DER "
+    " " NAME_VALUE(LTC_DER_MAX_RECURSION) " "
 #endif
 #if defined(LTC_FAST)
     " LTC_FAST "
Index: libtomcrypt-1.17/src/headers/tomcrypt_custom.h
===================================================================
--- libtomcrypt-1.17/src/headers/tomcrypt_custom.h
+++ libtomcrypt-1.17/src/headers/tomcrypt_custom.h	2018-07-21 21:20:37.484839080 +0200
@@ -350,7 +350,14 @@
    #define ECC521
 #endif
 #endif
+ 
+#if defined(LTC_DER)
+   #ifndef LTC_DER_MAX_RECURSION
+      /* Maximum recursion limit when processing nested ASN.1 types. */
+      #define LTC_DER_MAX_RECURSION 30
+   #endif
+#endif
 
 #if defined(LTC_MECC) || defined(LTC_MRSA) || defined(LTC_MDSA) || defined(MKATJA)
    /* Include the MPI functionality?  (required by the PK algorithms) */
    #define MPI
Index: libtomcrypt-1.17/src/pk/asn1/der/sequence/der_decode_sequence_flexi.c
===================================================================
--- libtomcrypt-1.17/src/pk/asn1/der/sequence/der_decode_sequence_flexi.c
+++ libtomcrypt-1.17/src/pk/asn1/der/sequence/der_decode_sequence_flexi.c	2018-07-21 21:28:18.765241002 +0200
@@ -62,8 +62,8 @@
 */   
 int der_decode_sequence_flexi(const unsigned char *in, unsigned long *inlen, ltc_asn1_list **out)
 {
-   ltc_asn1_list *l;
-   unsigned long err, type, len, totlen, x, y;
+   ltc_asn1_list *l, *t;
+   unsigned long err, type, len, totlen, x, y, len_len;
    void          *realloc_tmp;
    
    LTC_ARGCHK(in    != NULL);
@@ -338,7 +338,18 @@
              
              /* link them up y0 */
              l->child->parent = l;
-             
+
+             t = l;
+             len_len = 0;
+             while((t != NULL) && (t->child != NULL)) {
+                len_len++;
+                t = t->child;
+             }
+             if (len_len > LTC_DER_MAX_RECURSION) {
+                err = CRYPT_ERROR;
+                goto error;
+             }
+
              break;
          default:
            /* invalid byte ... this is a soft error */
Index: libtomcrypt-1.17/testprof/der_tests.c
===================================================================
--- libtomcrypt-1.17/testprof/der_tests.c
+++ libtomcrypt-1.17/testprof/der_tests.c	2018-07-21 21:33:19.281598263 +0200
@@ -507,6 +507,24 @@
   return 0;
 }
    
+static void _der_recursion_limit(void)
+{
+   int failed = 0;
+   unsigned int n;
+   unsigned long integer = 123, s;
+   ltc_asn1_list seqs[LTC_DER_MAX_RECURSION + 2], dummy[1], *flexi;
+   unsigned char buf[2048];
+   LTC_SET_ASN1(dummy, 0, LTC_ASN1_SHORT_INTEGER, &integer, 1);
+   LTC_SET_ASN1(seqs, LTC_DER_MAX_RECURSION + 1, LTC_ASN1_SEQUENCE, dummy, 1);
+   for (n = 0; n < LTC_DER_MAX_RECURSION + 1; ++n) {
+      LTC_SET_ASN1(seqs, LTC_DER_MAX_RECURSION - n, LTC_ASN1_SEQUENCE, &seqs[LTC_DER_MAX_RECURSION - n + 1], 1);
+   }
+   s = sizeof(buf);
+   DO(der_encode_sequence(seqs, 1, buf, &s));
+   DO(der_decode_sequence(buf, s, seqs, 1));
+   SHOULD_FAIL(der_decode_sequence_flexi(buf, &s, &flexi));
+   if (failed) exit(EXIT_FAILURE);
+}
 
 int der_tests(void)
 {
@@ -540,6 +558,8 @@
    unsigned char utf8_buf[32];
    wchar_t utf8_out[32];
 
+   _der_recursion_limit();
+
    DO(mp_init_multi(&a, &b, &c, &d, &e, &f, &g, NULL));
    for (zz = 0; zz < 16; zz++) {
 #ifdef USE_TFM
Index: libtomcrypt-1.17/testprof/tomcrypt_test.h
===================================================================
--- libtomcrypt-1.17/testprof/tomcrypt_test.h
+++ libtomcrypt-1.17/testprof/tomcrypt_test.h	2018-07-21 21:35:24.024087806 +0200
@@ -21,8 +21,10 @@
 
 #ifdef LTC_VERBOSE
 #define DO(x) do { fprintf(stderr, "%s:\n", #x); run_cmd((x), __LINE__, __FILE__, #x); } while (0);
+#define SHOULD_FAIL(x) do { fprintf(stderr, "%s:\n", #x); run_cmd((x) != CRYPT_OK ? CRYPT_OK : CRYPT_FAIL_TESTVECTOR, __LINE__, __FILE__, #x); } while (0)
 #else
 #define DO(x) do { run_cmd((x), __LINE__, __FILE__, #x); } while (0);
+#define SHOULD_FAIL(x) do { run_cmd((x) != CRYPT_OK ? CRYPT_OK : CRYPT_FAIL_TESTVECTOR, __LINE__, __FILE__, #x); } while (0)
 #endif
 
 /* TESTS */