From 04dd34c14ed21d36e80447dd988fa1ce4ebe5ac5 Mon Sep 17 00:00:00 2001
From: DRC <information@libjpeg-turbo.org>
Date: Thu, 4 Feb 2016 10:59:21 -0600
Subject: [PATCH] Guard against wrap-around in alloc functions

Because of the exposed nature of the libjpeg API, alloc_small() and
alloc_large() can potentially be called by external code.  If an
application were to call either of those functions with
sizeofobject > SIZE_MAX - ALIGN_SIZE - 1, then the math in
round_up_pow2() would wrap around to zero, causing that function to
return a small value.  That value would likely not exceed
MAX_ALLOC_CHUNK, so the subsequent size checks in alloc_small() and
alloc_large() would not catch the error.

A similar problem could occur in 32-bit builds if alloc_sarray() were
called with
samplesperrow > SIZE_MAX - (2 * ALIGN_SIZE / sizeof(JSAMPLE)) - 1

This patch simply ensures that the size argument to the alloc_*()
functions will never exceed MAX_ALLOC_CHUNK (1 billion).  If it did,
then subsequent size checks would eventually catch that error, so we
are instead catching the error before round_up_pow2() is called.

This addresses a minor concern (LJT-01-001) expressed in a security
audit by Cure53.
---
 jmemmgr.c | 16 ++++++++++++++++
 1 file changed, 16 insertions(+), 0 deletions(-)

diff --git a/jmemmgr.c b/jmemmgr.c
index 5fbebb4..4b0fcac 100644
--- a/jmemmgr.c
+++ b/jmemmgr.c
@@ -275,6 +275,11 @@ alloc_small (j_common_ptr cinfo, int pool_id, size_t sizeofobject)
    * and so that algorithms can straddle outside the proper area up
    * to the next alignment.
    */
+  if (sizeofobject > MAX_ALLOC_CHUNK) {
+    /* This prevents overflow/wrap-around in round_up_pow2() if sizeofobject
+       is close to SIZE_MAX. */
+    out_of_memory(cinfo, 7);
+  }
   sizeofobject = round_up_pow2(sizeofobject, ALIGN_SIZE);
 
   /* Check for unsatisfiable request (do now to ensure no overflow below) */
@@ -364,6 +369,11 @@ alloc_large (j_common_ptr cinfo, int pool_id, size_t sizeofobject)
    * algorithms can straddle outside the proper area up to the next
    * alignment.
    */
+  if (sizeofobject > MAX_ALLOC_CHUNK) {
+    /* This prevents overflow/wrap-around in round_up_pow2() if sizeofobject
+       is close to SIZE_MAX. */
+    out_of_memory(cinfo, 8);
+  }
   sizeofobject = round_up_pow2(sizeofobject, ALIGN_SIZE);
 
   /* Check for unsatisfiable request (do now to ensure no overflow below) */
@@ -431,6 +441,12 @@ alloc_sarray (j_common_ptr cinfo, int pool_id,
   /* Make sure each row is properly aligned */
   if ((ALIGN_SIZE % SIZEOF(JSAMPLE)) != 0)
     out_of_memory(cinfo, 5);	/* safety check */
+
+  if (samplesperrow > MAX_ALLOC_CHUNK) {
+    /* This prevents overflow/wrap-around in round_up_pow2() if sizeofobject
+       is close to SIZE_MAX. */
+    out_of_memory(cinfo, 9);
+  }
   samplesperrow = (JDIMENSION)round_up_pow2(samplesperrow, (2 * ALIGN_SIZE) / SIZEOF(JSAMPLE));
 
   /* Calculate max # of rows allowed in one allocation chunk */