check integer overflow in Matrix::setSize (#296)

* check integer multiplication overflow

* overflow check from a constructor function.

* style fix

* print error message correction

lib/linalg/Matrix.cpp

@@ -130,6 +130,10 @@ Matrix::Matrix(
         memcpy(d_mat, mat, d_alloc_size*sizeof(double));
     }
     else {
+        // Check integer multiplication overflow
+        if (num_rows > INT_MAX / num_cols)
+            CAROM_ERROR("Matrix::Matrix- new size exceeds maximum integer value!\n");
+
         d_mat = mat;
         d_alloc_size = num_rows*num_cols;
         d_num_cols = num_cols;

lib/linalg/Matrix.h

@@ -20,6 +20,7 @@
 #include <complex>
 #include <memory>
 #include <string>
+#include <climits>
 
 namespace CAROM {
 
@@ -149,6 +150,10 @@ class Matrix
         int num_rows,
         int num_cols)
     {
+        // Check integer multiplication overflow
+        if (num_rows > INT_MAX / num_cols)
+            CAROM_ERROR("Matrix::setSize- new size exceeds maximum integer value!\n");
+
         int new_size = num_rows*num_cols;
         if (new_size > d_alloc_size) {
             if (!d_owns_data) {