commit 2ad23b93b513d3a36af53d90dfc9ebfac80b4e3e
author Michael Niedermayer <michael@niedermayer.cc> Sun Feb 11 03:38:54 2018 +0100
committer Michael Niedermayer <michael@niedermayer.cc> Sun Feb 11 19:41:43 2018 +0100
tree e6f71a7c1b095216fa607d114de05cefae59e7b2
parent cec38382dbc8c876186c98c80823619ad6a6c41b

avcodec/vp3: Error out on invalid num_coeffs in unpack_vlcs()

This fixes a hypothetical integer overflow

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit f2318aee8ca8df1c84092f7d6691a2d0df02c474)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

libavcodec/vp3.c

diff --git a/libavcodec/vp3.c b/libavcodec/vp3.c
index fdffc88..e9bb7dc 100644
--- a/libavcodec/vp3.c
+++ b/libavcodec/vp3.c
@@ -951,9 +951,11 @@
     Vp3Fragment *all_fragments = s->all_fragments;
     VLC_TYPE(*vlc_table)[2] = table->table;
 
-    if (num_coeffs < 0)
+    if (num_coeffs < 0) {
         av_log(s->avctx, AV_LOG_ERROR,
                "Invalid number of coefficients at level %d\n", coeff_index);
+        return AVERROR_INVALIDDATA;
+    }
 
     if (eob_run > num_coeffs) {
         coeff_i      =