From bcc2421b4938fc1d9f5f3fb6ef2320571b27af42 Mon Sep 17 00:00:00 2001
From: nobu <nobu@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>
Date: Sun, 13 Dec 2015 09:35:58 +0000
Subject: [PATCH] function.c: check arguments

* ext/fiddle/function.c (initialize): check all arguments first.
  reported by Marcin 'Icewall' Noga of Cisco Talos.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@53082 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
---
 ChangeLog             |  5 ++++-
 ext/fiddle/function.c | 50 ++++++++++++++++++++++++++++++++++----------------
 2 files changed, 38 insertions(+), 17 deletions(-)

diff --git a/ext/fiddle/function.c b/ext/fiddle/function.c
index 772c09adcb46..a2760fa070de 100644
--- a/ext/fiddle/function.c
+++ b/ext/fiddle/function.c
@@ -15,12 +15,16 @@ VALUE cFiddleFunction;
 #define MAX_ARGS (SIZE_MAX / (sizeof(void *) + sizeof(fiddle_generic)) - 1)
 
 #define Check_Max_Args(name, len) \
+    Check_Max_Args_(name, len, "")
+#define Check_Max_Args_Long(name, len) \
+    Check_Max_Args_(name, len, "l")
+#define Check_Max_Args_(name, len, fmt) \
     if ((size_t)(len) < MAX_ARGS) { \
 	/* OK */ \
     } \
     else { \
 	rb_raise(rb_eTypeError, \
-		 name" is so large that it can cause integer overflow (%d)", \
+		 name" is so large that it can cause integer overflow (%"fmt"d)", \
 		 (len)); \
     }
 
@@ -87,16 +91,34 @@ static VALUE
 initialize(int argc, VALUE argv[], VALUE self)
 {
     ffi_cif * cif;
-    ffi_type **arg_types;
+    ffi_type **arg_types, *rtype;
     ffi_status result;
-    VALUE ptr, args, ret_type, abi, kwds;
-    int i;
+    VALUE ptr, args, ret_type, abi, kwds, ary;
+    int i, len;
+    int nabi;
+    void *cfunc;
 
     rb_scan_args(argc, argv, "31:", &ptr, &args, &ret_type, &abi, &kwds);
-    if(NIL_P(abi)) abi = INT2NUM(FFI_DEFAULT_ABI);
+    ptr = rb_Integer(ptr);
+    cfunc = NUM2PTR(ptr);
+    PTR2NUM(cfunc);
+    nabi = NIL_P(abi) ? FFI_DEFAULT_ABI : NUM2INT(abi);
+    abi = INT2FIX(nabi);
+    i = NUM2INT(ret_type);
+    rtype = INT2FFI_TYPE(i);
+    ret_type = INT2FIX(i);
 
     Check_Type(args, T_ARRAY);
-    Check_Max_Args("args", RARRAY_LENINT(args));
+    len = RARRAY_LENINT(args);
+    Check_Max_Args("args", len);
+    ary = rb_ary_subseq(args, 0, len);
+    for (i = 0; i < RARRAY_LEN(args); i++) {
+	VALUE a = RARRAY_PTR(args)[i];
+	int type = NUM2INT(a);
+	(void)INT2FFI_TYPE(type); /* raise */
+	if (INT2FIX(type) != a) rb_ary_store(ary, i, INT2FIX(type));
+    }
+    OBJ_FREEZE(ary);
 
     rb_iv_set(self, "@ptr", ptr);
     rb_iv_set(self, "@args", args);
@@ -107,20 +129,15 @@ initialize(int argc, VALUE argv[], VALUE self)
 
     TypedData_Get_Struct(self, ffi_cif, &function_data_type, cif);
 
-    arg_types = xcalloc(RARRAY_LEN(args) + 1, sizeof(ffi_type *));
+    arg_types = xcalloc(len + 1, sizeof(ffi_type *));
 
     for (i = 0; i < RARRAY_LEN(args); i++) {
 	int type = NUM2INT(RARRAY_PTR(args)[i]);
 	arg_types[i] = INT2FFI_TYPE(type);
     }
-    arg_types[RARRAY_LEN(args)] = NULL;
+    arg_types[len] = NULL;
 
-    result = ffi_prep_cif (
-	    cif,
-	    NUM2INT(abi),
-	    RARRAY_LENINT(args),
-	    INT2FFI_TYPE(NUM2INT(ret_type)),
-	    arg_types);
+    result = ffi_prep_cif(cif, nabi, len, rtype, arg_types);
 
     if (result)
 	rb_raise(rb_eRuntimeError, "error creating CIF %d", result);
@@ -182,8 +199,9 @@ function_call(int argc, VALUE argv[], VALUE self)
     for (i = 0; i < argc; i++) {
 	VALUE type = RARRAY_PTR(types)[i];
 	VALUE src = argv[i];
+	int argtype = FIX2INT(type);
 
-	if(NUM2INT(type) == TYPE_VOIDP) {
+	if (argtype == TYPE_VOIDP) {
 	    if(NIL_P(src)) {
 		src = INT2FIX(0);
 	    } else if(cPointer != CLASS_OF(src)) {
@@ -192,12 +210,12 @@ function_call(int argc, VALUE argv[], VALUE self)
 	    src = rb_Integer(src);
 	}
 
-	VALUE2GENERIC(NUM2INT(type), src, &generic_args[i]);
+	VALUE2GENERIC(argtype, src, &generic_args[i]);
 	values[i] = (void *)&generic_args[i];
     }
     values[argc] = NULL;
 
-    ffi_call(cif, NUM2PTR(rb_Integer(cfunc)), &retval, values);
+    ffi_call(cif, NUM2PTR(cfunc), &retval, values);
 
     rb_funcall(mFiddle, rb_intern("last_error="), 1, INT2NUM(errno));
 #if defined(_WIN32)