Check for overflow of crypto size

Bug: 111603051
Test: CTS
Change-Id: Ib5b1802b9b35769a25c16e2b977308cf7a810606
(cherry picked from commit d1fd02761236b35a336434367131f71bef7405c9)
diff --git a/media/ndk/NdkMediaCodec.cpp b/media/ndk/NdkMediaCodec.cpp
index 50b490d..73e733e 100644
--- a/media/ndk/NdkMediaCodec.cpp
+++ b/media/ndk/NdkMediaCodec.cpp
@@ -447,7 +447,13 @@
         size_t *encryptedbytes) {
 
     // size needed to store all the crypto data
-    size_t cryptosize = sizeof(AMediaCodecCryptoInfo) + sizeof(size_t) * numsubsamples * 2;
+    size_t cryptosize;
+    // = sizeof(AMediaCodecCryptoInfo) + sizeof(size_t) * numsubsamples * 2;
+    if (__builtin_mul_overflow(sizeof(size_t) * 2, numsubsamples, &cryptosize) ||
+            __builtin_add_overflow(cryptosize, sizeof(AMediaCodecCryptoInfo), &cryptosize)) {
+        ALOGE("crypto size overflow");
+        return NULL;
+    }
     AMediaCodecCryptoInfo *ret = (AMediaCodecCryptoInfo*) malloc(cryptosize);
     if (!ret) {
         ALOGE("couldn't allocate %zu bytes", cryptosize);