commit 2409f991667283f8fa1dacc66f245950693495aa
Author: Neil Fortner <nfortne2@hdfgroup.org>
Date:   Thu Sep 8 10:48:54 2016 -0500

    Fix issues in H5Znbit.c where the decompression algorithm would not check the compressed data for validity, potentially causing a buffer overflow.

Index: hdf5/src/H5Znbit.c
===================================================================
--- hdf5.orig/src/H5Znbit.c
+++ hdf5/src/H5Znbit.c
@@ -62,11 +62,11 @@ static void H5Z_nbit_decompress_one_noop
                        unsigned char *buffer, size_t *j, int *buf_len, unsigned size);
 static void H5Z_nbit_decompress_one_atomic(unsigned char *data, size_t data_offset,
                     unsigned char *buffer, size_t *j, int *buf_len, parms_atomic p);
-static void H5Z_nbit_decompress_one_array(unsigned char *data, size_t data_offset,
+static herr_t H5Z__nbit_decompress_one_array(unsigned char *data, size_t data_offset,
            unsigned char *buffer, size_t *j, int *buf_len, const unsigned parms[]);
-static void H5Z_nbit_decompress_one_compound(unsigned char *data, size_t data_offset,
+static herr_t H5Z__nbit_decompress_one_compound(unsigned char *data, size_t data_offset,
               unsigned char *buffer, size_t *j, int *buf_len, const unsigned parms[]);
-static void H5Z_nbit_decompress(unsigned char *data, unsigned d_nelmts, unsigned char *buffer,
+static herr_t H5Z__nbit_decompress(unsigned char *data, unsigned d_nelmts, unsigned char *buffer,
                                 const unsigned parms[]);
 static void H5Z_nbit_compress_one_nooptype(unsigned char *data, size_t data_offset,
                      unsigned char *buffer, size_t *j, int *buf_len, unsigned size);
@@ -899,7 +899,8 @@ H5Z_filter_nbit(unsigned flags, size_t c
             HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "memory allocation failed for nbit decompression")
 
         /* decompress the buffer */
-        H5Z_nbit_decompress(outbuf, d_nelmts, (unsigned char *)*buf, cd_values);
+        if(H5Z__nbit_decompress(outbuf, d_nelmts, (unsigned char *)*buf, cd_values) < 0)
+            HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, 0, "can't decompress buffer")
     } /* end if */
     /* output; compress */
     else {
@@ -1048,12 +1049,15 @@ H5Z_nbit_decompress_one_atomic(unsigned
    }
 }
 
-static void
-H5Z_nbit_decompress_one_array(unsigned char *data, size_t data_offset,
+static herr_t
+H5Z__nbit_decompress_one_array(unsigned char *data, size_t data_offset,
            unsigned char *buffer, size_t *j, int *buf_len, const unsigned parms[])
 {
    unsigned i, total_size, base_class, base_size, n, begin_index;
    parms_atomic p;
+   herr_t ret_value = SUCCEED; /* Return value */
+
+   FUNC_ENTER_STATIC
 
    total_size = parms[parms_index++];
    base_class = parms[parms_index++];
@@ -1064,6 +1068,11 @@ H5Z_nbit_decompress_one_array(unsigned c
            p.order = parms[parms_index++];
            p.precision = parms[parms_index++];
            p.offset = parms[parms_index++];
+
+           /* Check values of precision and offset */
+           if(p.precision > p.size * 8 || (p.precision + p.offset) > p.size * 8)
+              HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "invalid datatype precision/offset")
+
            n = total_size/p.size;
            for(i = 0; i < n; i++)
               H5Z_nbit_decompress_one_atomic(data, data_offset + i*p.size,
@@ -1074,8 +1083,9 @@ H5Z_nbit_decompress_one_array(unsigned c
            n = total_size/base_size; /* number of base_type elements inside the array datatype */
            begin_index = parms_index;
            for(i = 0; i < n; i++) {
-              H5Z_nbit_decompress_one_array(data, data_offset + i*base_size,
-                                            buffer, j, buf_len, parms);
+              if(H5Z__nbit_decompress_one_array(data, data_offset + i * base_size,
+                                            buffer, j, buf_len, parms) < 0)
+                 HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, FAIL, "can't decompress array")
               parms_index = begin_index;
            }
            break;
@@ -1084,8 +1094,9 @@ H5Z_nbit_decompress_one_array(unsigned c
            n = total_size/base_size; /* number of base_type elements inside the array datatype */
            begin_index = parms_index;
            for(i = 0; i < n; i++) {
-              H5Z_nbit_decompress_one_compound(data, data_offset + i*base_size,
-                                               buffer, j, buf_len, parms);
+              if(H5Z__nbit_decompress_one_compound(data, data_offset + i * base_size,
+                                               buffer, j, buf_len, parms) < 0)
+                 HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, FAIL, "can't decompress compound")
               parms_index = begin_index;
            }
            break;
@@ -1096,40 +1107,62 @@ H5Z_nbit_decompress_one_array(unsigned c
       default:
           HDassert(0 && "This Should never be executed!");
    } /* end switch */
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
 }
 
-static void
-H5Z_nbit_decompress_one_compound(unsigned char *data, size_t data_offset,
+static herr_t
+H5Z__nbit_decompress_one_compound(unsigned char *data, size_t data_offset,
               unsigned char *buffer, size_t *j, int *buf_len, const unsigned parms[])
 {
-   unsigned i, nmembers, member_offset, member_class, size;
+   unsigned i, nmembers, member_offset, member_class, member_size, used_size = 0, size;
    parms_atomic p;
+   herr_t ret_value = SUCCEED; /* Return value */
+
+   FUNC_ENTER_STATIC
 
-   parms_index++; /* skip total size of compound datatype */
+   size = parms[parms_index++];
    nmembers = parms[parms_index++];
 
    for(i = 0; i < nmembers; i++) {
       member_offset = parms[parms_index++];
       member_class = parms[parms_index++];
+
+      /* Check for overflow */
+      member_size = parms[parms_index];
+      used_size += member_size;
+      if(used_size > size)
+         HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "compound member offset overflowed compound size")
       switch(member_class) {
          case H5Z_NBIT_ATOMIC:
-              p.size = parms[parms_index++];
+              p.size = member_size;
+              /* Advance past member size */
+              parms_index++;
               p.order = parms[parms_index++];
               p.precision = parms[parms_index++];
               p.offset = parms[parms_index++];
+
+              /* Check values of precision and offset */
+              if(p.precision > p.size * 8 || (p.precision + p.offset) > p.size * 8)
+                 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "invalid datatype precision/offset")
+
               H5Z_nbit_decompress_one_atomic(data, data_offset + member_offset,
                                              buffer, j, buf_len, p);
               break;
          case H5Z_NBIT_ARRAY:
-              H5Z_nbit_decompress_one_array(data, data_offset + member_offset,
-                                            buffer, j, buf_len, parms);
+              if(H5Z__nbit_decompress_one_array(data, data_offset + member_offset,
+                                            buffer, j, buf_len, parms) < 0)
+                 HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, FAIL, "can't decompress array")
               break;
          case H5Z_NBIT_COMPOUND:
-              H5Z_nbit_decompress_one_compound(data, data_offset+member_offset,
-                                               buffer, j, buf_len, parms);
+              if(H5Z__nbit_decompress_one_compound(data, data_offset+member_offset,
+                                            buffer, j, buf_len, parms) < 0)
+                 HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, FAIL, "can't decompress compound")
               break;
          case H5Z_NBIT_NOOPTYPE:
-              size = parms[parms_index++];
+              /* Advance past member size */
+              parms_index++;
               H5Z_nbit_decompress_one_nooptype(data, data_offset+member_offset,
                                                buffer, j, buf_len, size);
               break;
@@ -1137,10 +1170,13 @@ H5Z_nbit_decompress_one_compound(unsigne
               HDassert(0 && "This Should never be executed!");
       } /* end switch */
    }
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
 }
 
-static void
-H5Z_nbit_decompress(unsigned char *data, unsigned d_nelmts, unsigned char *buffer,
+static herr_t
+H5Z__nbit_decompress(unsigned char *data, unsigned d_nelmts, unsigned char *buffer,
                                 const unsigned parms[])
 {
    /* i: index of data, j: index of buffer,
@@ -1148,6 +1184,9 @@ H5Z_nbit_decompress(unsigned char *data,
    size_t i, j, size;
    int buf_len;
    parms_atomic p;
+   herr_t ret_value = SUCCEED; /* Return value */
+
+   FUNC_ENTER_STATIC
 
    /* may not have to initialize to zeros */
    for(i = 0; i < d_nelmts*parms[4]; i++) data[i] = 0;
@@ -1163,6 +1202,11 @@ H5Z_nbit_decompress(unsigned char *data,
            p.order = parms[5];
            p.precision = parms[6];
            p.offset = parms[7];
+
+           /* Check values of precision and offset */
+           if(p.precision > p.size * 8 || (p.precision + p.offset) > p.size * 8)
+              HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "invalid datatype precision/offset")
+
            for(i = 0; i < d_nelmts; i++) {
               H5Z_nbit_decompress_one_atomic(data, i*p.size, buffer, &j, &buf_len, p);
            }
@@ -1171,7 +1215,8 @@ H5Z_nbit_decompress(unsigned char *data,
            size = parms[4];
            parms_index = 4;
            for(i = 0; i < d_nelmts; i++) {
-              H5Z_nbit_decompress_one_array(data, i*size, buffer, &j, &buf_len, parms);
+              if(H5Z__nbit_decompress_one_array(data, i * size, buffer, &j, &buf_len, parms) < 0)
+                  HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, FAIL, "can't decompress array")
               parms_index = 4;
            }
            break;
@@ -1179,13 +1224,17 @@ H5Z_nbit_decompress(unsigned char *data,
            size = parms[4];
            parms_index = 4;
            for(i = 0; i < d_nelmts; i++) {
-              H5Z_nbit_decompress_one_compound(data, i*size, buffer, &j, &buf_len, parms);
+              if(H5Z__nbit_decompress_one_compound(data, i * size, buffer, &j, &buf_len, parms) < 0)
+                  HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, FAIL, "can't decompress compound")
               parms_index = 4;
            }
            break;
       default:
           HDassert(0 && "This Should never be executed!");
    } /* end switch */
+
+done:
+   FUNC_LEAVE_NOAPI(ret_value)
 }
 
 static void H5Z_nbit_compress_one_byte(unsigned char *data, size_t data_offset, int k, int begin_i,