tiff: Security fix CVE-2016-9535

* libtiff/tif_predict.h, libtiff/tif_predict.c:
Replace assertions by runtime checks to avoid assertions in debug mode,
or buffer overflows in release mode. Can happen when dealing with
unusual tile size like YCbCr with subsampling.

External References:
http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2016-9535

Patch from:
https://github.com/vadz/libtiff/commit/3ca657a8793dd011bf869695d72ad31c779c3cc1
https://github.com/vadz/libtiff/commit/6a984bf7905c6621281588431f384e79d11a2e33

(From OE-Core rev: 61d3feb9cad9f61f6551b43f4f19bfa33cadd275)

Signed-off-by: Mingli Yu <Mingli.Yu@windriver.com>
Signed-off-by: Ross Burton <ross.burton@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Signed-off-by: Armin Kuster <akuster808@gmail.com>

3 files changed, 492 insertions, 0 deletions

diff --git a/meta/recipes-multimedia/libtiff/files/CVE-2016-9535-1.patch b/meta/recipes-multimedia/libtiff/files/CVE-2016-9535-1.patch
new file mode 100644
index 0000000000..26fd0df11c
--- /dev/null
+++ b/meta/recipes-multimedia/libtiff/files/CVE-2016-9535-1.patch
@@ -0,0 +1,423 @@
+From 3ca657a8793dd011bf869695d72ad31c779c3cc1 Mon Sep 17 00:00:00 2001
+From: erouault <erouault>
+Date: Mon, 31 Oct 2016 17:24:26 +0000
+Subject: [PATCH 1/2] Fix CVE-2016-9535
+
+* libtiff/tif_predict.h, libtiff/tif_predict.c: Replace
+ assertions by runtime checks to avoid assertions in debug mode, or buffer
+ overflows in release mode. Can happen when dealing with unusual tile size
+ like YCbCr with subsampling. Reported as MSVR 35105 by Axel Souchet	&
+ Vishal Chauhan from the MSRC Vulnerabilities & Mitigations team.
+
+CVE: CVE-2016-9535
+Upstream-Status: Backport
+https://github.com/vadz/libtiff/commit/3ca657a8793dd011bf869695d72ad31c779c3cc1
+
+Signed-off-by: Mingli Yu <Mingli.Yu@windriver.com>
+
+---
+ libtiff/tif_predict.c | 153 +++++++++++++++++++++++++++++++++++---------------
+ libtiff/tif_predict.h |   6 +-
+ 2 files changed, 121 insertions(+), 47 deletions(-)
+
+diff --git a/libtiff/tif_predict.c b/libtiff/tif_predict.c
+index 555f2f9..b829259 100644
+--- a/libtiff/tif_predict.c
++++ b/libtiff/tif_predict.c
+@@ -34,18 +34,18 @@
+ 
+ #define	PredictorState(tif)	((TIFFPredictorState*) (tif)->tif_data)
+ 
+-static void horAcc8(TIFF* tif, uint8* cp0, tmsize_t cc);
+-static void horAcc16(TIFF* tif, uint8* cp0, tmsize_t cc);
+-static void horAcc32(TIFF* tif, uint8* cp0, tmsize_t cc);
+-static void swabHorAcc16(TIFF* tif, uint8* cp0, tmsize_t cc);
+-static void swabHorAcc32(TIFF* tif, uint8* cp0, tmsize_t cc);
+-static void horDiff8(TIFF* tif, uint8* cp0, tmsize_t cc);
+-static void horDiff16(TIFF* tif, uint8* cp0, tmsize_t cc);
+-static void horDiff32(TIFF* tif, uint8* cp0, tmsize_t cc);
+-static void swabHorDiff16(TIFF* tif, uint8* cp0, tmsize_t cc);
+-static void swabHorDiff32(TIFF* tif, uint8* cp0, tmsize_t cc);
+-static void fpAcc(TIFF* tif, uint8* cp0, tmsize_t cc);
+-static void fpDiff(TIFF* tif, uint8* cp0, tmsize_t cc);
++static int horAcc8(TIFF* tif, uint8* cp0, tmsize_t cc);
++static int horAcc16(TIFF* tif, uint8* cp0, tmsize_t cc);
++static int horAcc32(TIFF* tif, uint8* cp0, tmsize_t cc);
++static int swabHorAcc16(TIFF* tif, uint8* cp0, tmsize_t cc);
++static int swabHorAcc32(TIFF* tif, uint8* cp0, tmsize_t cc);
++static int horDiff8(TIFF* tif, uint8* cp0, tmsize_t cc);
++static int horDiff16(TIFF* tif, uint8* cp0, tmsize_t cc);
++static int horDiff32(TIFF* tif, uint8* cp0, tmsize_t cc);
++static int swabHorDiff16(TIFF* tif, uint8* cp0, tmsize_t cc);
++static int swabHorDiff32(TIFF* tif, uint8* cp0, tmsize_t cc);
++static int fpAcc(TIFF* tif, uint8* cp0, tmsize_t cc);
++static int fpDiff(TIFF* tif, uint8* cp0, tmsize_t cc);
+ static int PredictorDecodeRow(TIFF* tif, uint8* op0, tmsize_t occ0, uint16 s);
+ static int PredictorDecodeTile(TIFF* tif, uint8* op0, tmsize_t occ0, uint16 s);
+ static int PredictorEncodeRow(TIFF* tif, uint8* bp, tmsize_t cc, uint16 s);
+@@ -273,13 +273,19 @@ PredictorSetupEncode(TIFF* tif)
+ /* - when storing into the byte stream, we explicitly mask with 0xff so */
+ /*   as to make icc -check=conversions happy (not necessary by the standard) */
+ 
+-static void
++static int
+ horAcc8(TIFF* tif, uint8* cp0, tmsize_t cc)
+ {
+ 	tmsize_t stride = PredictorState(tif)->stride;
+ 
+ 	unsigned char* cp = (unsigned char*) cp0;
+-	assert((cc%stride)==0);
++    if((cc%stride)!=0)
++    {
++        TIFFErrorExt(tif->tif_clientdata, "horAcc8",
++                     "%s", "(cc%stride)!=0");
++        return 0;
++    }
++
+ 	if (cc > stride) {
+ 		/*
+ 		 * Pipeline the most common cases.
+@@ -321,26 +327,32 @@ horAcc8(TIFF* tif, uint8* cp0, tmsize_t cc)
+ 			} while (cc>0);
+ 		}
+ 	}
++	return 1;
+ }
+ 
+-static void
++static int
+ swabHorAcc16(TIFF* tif, uint8* cp0, tmsize_t cc)
+ {
+ 	uint16* wp = (uint16*) cp0;
+ 	tmsize_t wc = cc / 2;
+ 
+         TIFFSwabArrayOfShort(wp, wc);
+-        horAcc16(tif, cp0, cc);
++        return horAcc16(tif, cp0, cc);
+ }
+ 
+-static void
++static int
+ horAcc16(TIFF* tif, uint8* cp0, tmsize_t cc)
+ {
+ 	tmsize_t stride = PredictorState(tif)->stride;
+ 	uint16* wp = (uint16*) cp0;
+ 	tmsize_t wc = cc / 2;
+ 
+-	assert((cc%(2*stride))==0);
++    if((cc%(2*stride))!=0)
++    {
++        TIFFErrorExt(tif->tif_clientdata, "horAcc16",
++                     "%s", "cc%(2*stride))!=0");
++        return 0;
++    }
+ 
+ 	if (wc > stride) {
+ 		wc -= stride;
+@@ -349,26 +361,32 @@ horAcc16(TIFF* tif, uint8* cp0, tmsize_t cc)
+ 			wc -= stride;
+ 		} while (wc > 0);
+ 	}
++	return 1;
+ }
+ 
+-static void
++static int
+ swabHorAcc32(TIFF* tif, uint8* cp0, tmsize_t cc)
+ {
+ 	uint32* wp = (uint32*) cp0;
+ 	tmsize_t wc = cc / 4;
+ 
+         TIFFSwabArrayOfLong(wp, wc);
+-	horAcc32(tif, cp0, cc);
++	return horAcc32(tif, cp0, cc);
+ }
+ 
+-static void
++static int
+ horAcc32(TIFF* tif, uint8* cp0, tmsize_t cc)
+ {
+ 	tmsize_t stride = PredictorState(tif)->stride;
+ 	uint32* wp = (uint32*) cp0;
+ 	tmsize_t wc = cc / 4;
+ 
+-	assert((cc%(4*stride))==0);
++    if((cc%(4*stride))!=0)
++    {
++        TIFFErrorExt(tif->tif_clientdata, "horAcc32",
++                     "%s", "cc%(4*stride))!=0");
++        return 0;
++    }
+ 
+ 	if (wc > stride) {
+ 		wc -= stride;
+@@ -377,12 +395,13 @@ horAcc32(TIFF* tif, uint8* cp0, tmsize_t cc)
+ 			wc -= stride;
+ 		} while (wc > 0);
+ 	}
++	return 1;
+ }
+ 
+ /*
+  * Floating point predictor accumulation routine.
+  */
+-static void
++static int
+ fpAcc(TIFF* tif, uint8* cp0, tmsize_t cc)
+ {
+ 	tmsize_t stride = PredictorState(tif)->stride;
+@@ -392,10 +411,15 @@ fpAcc(TIFF* tif, uint8* cp0, tmsize_t cc)
+ 	uint8 *cp = (uint8 *) cp0;
+ 	uint8 *tmp = (uint8 *)_TIFFmalloc(cc);
+ 
+-	assert((cc%(bps*stride))==0);
++    if(cc%(bps*stride)!=0)
++    {
++        TIFFErrorExt(tif->tif_clientdata, "fpAcc",
++                     "%s", "cc%(bps*stride))!=0");
++        return 0;
++    }
+ 
+ 	if (!tmp)
+-		return;
++		return 0;
+ 
+ 	while (count > stride) {
+ 		REPEAT4(stride, cp[stride] =
+@@ -417,6 +441,7 @@ fpAcc(TIFF* tif, uint8* cp0, tmsize_t cc)
+ 		}
+ 	}
+ 	_TIFFfree(tmp);
++    return 1;
+ }
+ 
+ /*
+@@ -432,8 +457,7 @@ PredictorDecodeRow(TIFF* tif, uint8* op0, tmsize_t occ0, uint16 s)
+ 	assert(sp->decodepfunc != NULL);  
+ 
+ 	if ((*sp->decoderow)(tif, op0, occ0, s)) {
+-		(*sp->decodepfunc)(tif, op0, occ0);
+-		return 1;
++		return (*sp->decodepfunc)(tif, op0, occ0);
+ 	} else
+ 		return 0;
+ }
+@@ -456,10 +480,16 @@ PredictorDecodeTile(TIFF* tif, uint8* op0, tmsize_t occ0, uint16 s)
+ 	if ((*sp->decodetile)(tif, op0, occ0, s)) {
+ 		tmsize_t rowsize = sp->rowsize;
+ 		assert(rowsize > 0);
+-		assert((occ0%rowsize)==0);
++		if((occ0%rowsize) !=0)
++        {
++            TIFFErrorExt(tif->tif_clientdata, "PredictorDecodeTile",
++                         "%s", "occ0%rowsize != 0");
++            return 0;
++        }
+ 		assert(sp->decodepfunc != NULL);
+ 		while (occ0 > 0) {
+-			(*sp->decodepfunc)(tif, op0, rowsize);
++			if( !(*sp->decodepfunc)(tif, op0, rowsize) )
++                return 0;
+ 			occ0 -= rowsize;
+ 			op0 += rowsize;
+ 		}
+@@ -468,14 +498,19 @@ PredictorDecodeTile(TIFF* tif, uint8* op0, tmsize_t occ0, uint16 s)
+ 		return 0;
+ }
+ 
+-static void
++static int
+ horDiff8(TIFF* tif, uint8* cp0, tmsize_t cc)
+ {
+ 	TIFFPredictorState* sp = PredictorState(tif);
+ 	tmsize_t stride = sp->stride;
+ 	unsigned char* cp = (unsigned char*) cp0;
+ 
+-	assert((cc%stride)==0);
++    if((cc%stride)!=0)
++    {
++        TIFFErrorExt(tif->tif_clientdata, "horDiff8",
++                     "%s", "(cc%stride)!=0");
++        return 0;
++    }
+ 
+ 	if (cc > stride) {
+ 		cc -= stride;
+@@ -513,9 +548,10 @@ horDiff8(TIFF* tif, uint8* cp0, tmsize_t cc)
+ 			} while ((cc -= stride) > 0);
+ 		}
+ 	}
++	return 1;
+ }
+ 
+-static void
++static int
+ horDiff16(TIFF* tif, uint8* cp0, tmsize_t cc)
+ {
+ 	TIFFPredictorState* sp = PredictorState(tif);
+@@ -523,7 +559,12 @@ horDiff16(TIFF* tif, uint8* cp0, tmsize_t cc)
+ 	uint16 *wp = (uint16*) cp0;
+ 	tmsize_t wc = cc/2;
+ 
+-	assert((cc%(2*stride))==0);
++    if((cc%(2*stride))!=0)
++    {
++        TIFFErrorExt(tif->tif_clientdata, "horDiff8",
++                     "%s", "(cc%(2*stride))!=0");
++        return 0;
++    }
+ 
+ 	if (wc > stride) {
+ 		wc -= stride;
+@@ -533,20 +574,23 @@ horDiff16(TIFF* tif, uint8* cp0, tmsize_t cc)
+ 			wc -= stride;
+ 		} while (wc > 0);
+ 	}
++	return 1;
+ }
+ 
+-static void
++static int
+ swabHorDiff16(TIFF* tif, uint8* cp0, tmsize_t cc)
+ {
+     uint16* wp = (uint16*) cp0;
+     tmsize_t wc = cc / 2;
+ 
+-    horDiff16(tif, cp0, cc);
++    if( !horDiff16(tif, cp0, cc) )
++        return 0;
+ 
+     TIFFSwabArrayOfShort(wp, wc);
++    return 1;
+ }
+ 
+-static void
++static int
+ horDiff32(TIFF* tif, uint8* cp0, tmsize_t cc)
+ {
+ 	TIFFPredictorState* sp = PredictorState(tif);
+@@ -554,7 +598,12 @@ horDiff32(TIFF* tif, uint8* cp0, tmsize_t cc)
+ 	uint32 *wp = (uint32*) cp0;
+ 	tmsize_t wc = cc/4;
+ 
+-	assert((cc%(4*stride))==0);
++    if((cc%(4*stride))!=0)
++    {
++        TIFFErrorExt(tif->tif_clientdata, "horDiff32",
++                     "%s", "(cc%(4*stride))!=0");
++        return 0;
++    }
+ 
+ 	if (wc > stride) {
+ 		wc -= stride;
+@@ -564,23 +613,26 @@ horDiff32(TIFF* tif, uint8* cp0, tmsize_t cc)
+ 			wc -= stride;
+ 		} while (wc > 0);
+ 	}
++	return 1;
+ }
+ 
+-static void
++static int
+ swabHorDiff32(TIFF* tif, uint8* cp0, tmsize_t cc)
+ {
+     uint32* wp = (uint32*) cp0;
+     tmsize_t wc = cc / 4;
+ 
+-    horDiff32(tif, cp0, cc);
++    if( !horDiff32(tif, cp0, cc) )
++        return 0;
+ 
+     TIFFSwabArrayOfLong(wp, wc);
++    return 1;
+ }
+ 
+ /*
+  * Floating point predictor differencing routine.
+  */
+-static void
++static int
+ fpDiff(TIFF* tif, uint8* cp0, tmsize_t cc)
+ {
+ 	tmsize_t stride = PredictorState(tif)->stride;
+@@ -590,10 +642,14 @@ fpDiff(TIFF* tif, uint8* cp0, tmsize_t cc)
+ 	uint8 *cp = (uint8 *) cp0;
+ 	uint8 *tmp = (uint8 *)_TIFFmalloc(cc);
+ 
+-	assert((cc%(bps*stride))==0);
+-
++    if((cc%(bps*stride))!=0)
++    {
++        TIFFErrorExt(tif->tif_clientdata, "fpDiff",
++                     "%s", "(cc%(bps*stride))!=0");
++        return 0;
++    }
+ 	if (!tmp)
+-		return;
++		return 0;
+ 
+ 	_TIFFmemcpy(tmp, cp0, cc);
+ 	for (count = 0; count < wc; count++) {
+@@ -613,6 +669,7 @@ fpDiff(TIFF* tif, uint8* cp0, tmsize_t cc)
+ 	cp += cc - stride - 1;
+ 	for (count = cc; count > stride; count -= stride)
+ 		REPEAT4(stride, cp[stride] = (unsigned char)((cp[stride] - cp[0])&0xff); cp--)
++    return 1;
+ }
+ 
+ static int
+@@ -625,7 +682,8 @@ PredictorEncodeRow(TIFF* tif, uint8* bp, tmsize_t cc, uint16 s)
+ 	assert(sp->encoderow != NULL);
+ 
+ 	/* XXX horizontal differencing alters user's data XXX */
+-	(*sp->encodepfunc)(tif, bp, cc);
++	if( !(*sp->encodepfunc)(tif, bp, cc) )
++        return 0;
+ 	return (*sp->encoderow)(tif, bp, cc, s);
+ }
+ 
+@@ -660,7 +718,12 @@ PredictorEncodeTile(TIFF* tif, uint8* bp0, tmsize_t cc0, uint16 s)
+ 
+ 	rowsize = sp->rowsize;
+ 	assert(rowsize > 0);
+-	assert((cc0%rowsize)==0);
++	if((cc0%rowsize)!=0)
++    {
++        TIFFErrorExt(tif->tif_clientdata, "PredictorEncodeTile",
++                     "%s", "(cc0%rowsize)!=0");
++        return 0;
++    }
+ 	while (cc > 0) {
+ 		(*sp->encodepfunc)(tif, bp, rowsize);
+ 		cc -= rowsize;
+diff --git a/libtiff/tif_predict.h b/libtiff/tif_predict.h
+index 91330cc..9e485a4 100644
+--- a/libtiff/tif_predict.h
++++ b/libtiff/tif_predict.h
+@@ -30,6 +30,8 @@
+  * ``Library-private'' Support for the Predictor Tag
+  */
+ 
++typedef int (*TIFFEncodeDecodeMethod)(TIFF* tif, uint8* buf, tmsize_t size);
++
+ /*
+  * Codecs that want to support the Predictor tag must place
+  * this structure first in their private state block so that
+@@ -43,12 +45,12 @@ typedef struct {
+ 	TIFFCodeMethod  encoderow;	/* parent codec encode/decode row */
+ 	TIFFCodeMethod  encodestrip;	/* parent codec encode/decode strip */
+ 	TIFFCodeMethod  encodetile;	/* parent codec encode/decode tile */ 
+-	TIFFPostMethod  encodepfunc;	/* horizontal differencer */
++	TIFFEncodeDecodeMethod  encodepfunc;	/* horizontal differencer */
+ 
+ 	TIFFCodeMethod  decoderow;	/* parent codec encode/decode row */
+ 	TIFFCodeMethod  decodestrip;	/* parent codec encode/decode strip */
+ 	TIFFCodeMethod  decodetile;	/* parent codec encode/decode tile */ 
+-	TIFFPostMethod  decodepfunc;	/* horizontal accumulator */
++	TIFFEncodeDecodeMethod  decodepfunc;	/* horizontal accumulator */
+ 
+ 	TIFFVGetMethod  vgetparent;	/* super-class method */
+ 	TIFFVSetMethod  vsetparent;	/* super-class method */
+-- 
+2.9.3
+
diff --git a/meta/recipes-multimedia/libtiff/files/CVE-2016-9535-2.patch b/meta/recipes-multimedia/libtiff/files/CVE-2016-9535-2.patch
new file mode 100644
index 0000000000..977dbf6c87
--- /dev/null
+++ b/meta/recipes-multimedia/libtiff/files/CVE-2016-9535-2.patch
@@ -0,0 +1,67 @@
+From 6a984bf7905c6621281588431f384e79d11a2e33 Mon Sep 17 00:00:00 2001
+From: erouault <erouault>
+Date: Fri, 4 Nov 2016 09:19:13 +0000
+Subject: [PATCH 2/2] Fix CVE-2016-9535
+* libtiff/tif_predic.c: fix memory leaks in error code
+ paths added in previous commit (fix for MSVR 35105)
+
+CVE: CVE-2016-9535
+Upstream-Status: Backport
+https://github.com/vadz/libtiff/commit/6a984bf7905c6621281588431f384e79d11a2e33
+
+Signed-off-by: Mingli Yu <Mingli.Yu@windriver.com>
+
+---
+ libtiff/tif_predict.c | 8 ++++++--
+ 1 files changed, 11 insertions(+), 2 deletions(-)
+
+diff --git a/libtiff/tif_predict.c b/libtiff/tif_predict.c
+index b829259..3f42f3b 100644
+--- a/libtiff/tif_predict.c
++++ b/libtiff/tif_predict.c
+@@ -409,7 +409,7 @@ fpAcc(TIFF* tif, uint8* cp0, tmsize_t cc)
+ 	tmsize_t wc = cc / bps;
+ 	tmsize_t count = cc;
+ 	uint8 *cp = (uint8 *) cp0;
+-	uint8 *tmp = (uint8 *)_TIFFmalloc(cc);
++	uint8 *tmp;
+ 
+     if(cc%(bps*stride)!=0)
+     {
+@@ -418,6 +418,7 @@ fpAcc(TIFF* tif, uint8* cp0, tmsize_t cc)
+         return 0;
+     }
+ 
++    tmp = (uint8 *)_TIFFmalloc(cc);
+ 	if (!tmp)
+ 		return 0;
+ 
+@@ -640,7 +641,7 @@ fpDiff(TIFF* tif, uint8* cp0, tmsize_t cc)
+ 	tmsize_t wc = cc / bps;
+ 	tmsize_t count;
+ 	uint8 *cp = (uint8 *) cp0;
+-	uint8 *tmp = (uint8 *)_TIFFmalloc(cc);
++	uint8 *tmp;
+ 
+     if((cc%(bps*stride))!=0)
+     {
+@@ -648,6 +649,8 @@ fpDiff(TIFF* tif, uint8* cp0, tmsize_t cc)
+                      "%s", "(cc%(bps*stride))!=0");
+         return 0;
+     }
++
++    tmp = (uint8 *)_TIFFmalloc(cc);
+ 	if (!tmp)
+ 		return 0;
+ 
+@@ -722,6 +725,7 @@ PredictorEncodeTile(TIFF* tif, uint8* bp0, tmsize_t cc0, uint16 s)
+     {
+         TIFFErrorExt(tif->tif_clientdata, "PredictorEncodeTile",
+                      "%s", "(cc0%rowsize)!=0");
++        _TIFFfree( working_copy );
+         return 0;
+     }
+ 	while (cc > 0) {
+-- 
+2.9.3
+
diff --git a/meta/recipes-multimedia/libtiff/tiff_4.0.6.bb b/meta/recipes-multimedia/libtiff/tiff_4.0.6.bb
index a6f714c4b5..6495d1fad5 100644
--- a/meta/recipes-multimedia/libtiff/tiff_4.0.6.bb
+++ b/meta/recipes-multimedia/libtiff/tiff_4.0.6.bb
@@ -21,6 +21,8 @@ SRC_URI = "http://download.osgeo.org/libtiff/tiff-${PV}.tar.gz \
            file://CVE-2016-3632.patch \
            file://CVE-2016-9540.patch \
            file://CVE-2016-9539.patch \
+           file://CVE-2016-9535-1.patch \
+           file://CVE-2016-9535-2.patch \
           "
 
 SRC_URI[md5sum] = "d1d2e940dea0b5ad435f21f03d96dd72"