[RESTRICT AUTOMERGE] Prevent integer overflows when handling large images

This is a backport of an upstream patch. Original commit message:

"""
Prevent several integer overflow issues and subsequent segfaults that
occurred when attempting to compress or decompress gigapixel images with
the TurboJPEG API:

- Modify tjBufSize(), tjBufSizeYUV2(), and tjPlaneSizeYUV() to avoid
  integer overflow when computing the return values and to return an
  error if such an overflow is unavoidable.
- Modify tjunittest to validate the above.
- Modify tjCompress2(), tjEncodeYUVPlanes(), tjDecompress2(), and
  tjDecodeYUVPlanes() to avoid integer overflow when computing the row
  pointers in the 64-bit TurboJPEG C API.
- Modify TJBench (both C and Java versions) to avoid overflowing the
  size argument to malloc()/new and to fail gracefully if such an
  overflow is unavoidable.

In general, this allows gigapixel images to be accommodated by the
64-bit TurboJPEG C API when using automatic JPEG buffer (re)allocation.
Such images cannot currently be accommodated without automatic JPEG
buffer (re)allocation, due to the fact that tjAlloc() accepts a 32-bit
integer argument (oops.)  Such images cannot be accommodated in the
TurboJPEG Java API due to the fact that Java always uses a signed 32-bit
integer as an array index.
"""

Bug: 120551338
Test: tj64 /data/local/tmp/crash-46.jpg
See b/120551338#comment1 to get crash-46.jpg

Change-Id: I3a4e772c6017ffeb2181510f8e447f7e91baa095
(cherry picked from commit b1ab8ba6d91e0909606fdd386e7c2a64b82098a9)
diff --git a/README.android b/README.android
index 0fa0175..a6d00d3 100644
--- a/README.android
+++ b/README.android
@@ -34,3 +34,9 @@
 Fix sign mismatch comparison warnings
 Cherry picked from upstream:
 https://github.com/libjpeg-turbo/libjpeg-turbo/commit/d22fd541bf9dd87889c25909e19a640a580bcad7
+
+(5) java/TJBench.java, tjbench.c, tjunittest.c, and turbojpeg.c
+Add checks to ensure that the image is not larger than the allocated buffers.
+Cherry picked from upstream:
+https://github.com/libjpeg-turbo/libjpeg-turbo/commit/2a9e3bd7430cfda1bc812d139e0609c6aca0b884
+
diff --git a/java/TJBench.java b/java/TJBench.java
index 19db789..383be46 100644
--- a/java/TJBench.java
+++ b/java/TJBench.java
@@ -96,6 +96,8 @@
     int rindex = TJ.getRedOffset(pixelFormat);
     int gindex = TJ.getGreenOffset(pixelFormat);
     int bindex = TJ.getBlueOffset(pixelFormat);
+    if ((long)w[0] * (long)h[0] * (long)ps > (long)Integer.MAX_VALUE)
+      throw new Exception("Image is too lange");
     byte[] dstBuf = new byte[w[0] * h[0] * ps];
     int pixels = w[0] * h[0], dstPtr = 0, rgbPtr = 0;
     while (pixels-- > 0) {
@@ -147,8 +149,11 @@
 
     tjd = new TJDecompressor();
 
-    if (dstBuf == null)
+    if (dstBuf == null) {
+      if ((long)pitch * (long)scaledh > (long)Integer.MAX_VALUE)
+          throw new Exception("Image is too large");
       dstBuf = new byte[pitch * scaledh];
+    }
 
     /* Set the destination buffer to gray so we know whether the decompressor
        attempted to write to it */
@@ -284,6 +289,8 @@
     String pfStr = pixFormatStr[pf];
     YUVImage yuvImage = null;
 
+    if ((long)pitch * (long)h > (long)Integer.MAX_VALUE)
+        throw new Exception("Image is too lange");
     tmpBuf = new byte[pitch * h];
 
     if (quiet == 0)
@@ -429,6 +436,8 @@
     int ps = TJ.getPixelSize(pf), tile;
 
     FileInputStream fis = new FileInputStream(fileName);
+    if (fis.getChannel().size() > (long)Integer.MAX_VALUE)
+      throw new Exception("Image is too large");
     int srcSize = (int)fis.getChannel().size();
     srcBuf = new byte[srcSize];
     fis.read(srcBuf, 0, srcSize);
diff --git a/tjbench.c b/tjbench.c
index 9db1968..c62582e 100644
--- a/tjbench.c
+++ b/tjbench.c
@@ -127,7 +127,10 @@
 
 	if(dstbuf==NULL)
 	{
-		if((dstbuf=(unsigned char *)malloc(pitch*scaledh))==NULL)
+		if ((unsigned long long)pitch * (unsigned long long)scaledh >
+		    (unsigned long long)((size_t)-1))
+			_throw("allocating destination buffer", "Image is too large");
+		if((dstbuf=(unsigned char *)malloc((size_t)pitch*scaledh))==NULL)
 			_throwunix("allocating destination buffer");
 		dstbufalloc=1;
 	}
@@ -139,7 +142,10 @@
 	{
 		int width=dotile? tilew:scaledw;
 		int height=dotile? tileh:scaledh;
-		int yuvsize=tjBufSizeYUV2(width, yuvpad, height, subsamp);
+		unsigned long yuvsize=tjBufSizeYUV2(width, yuvpad, height, subsamp);
+
+		if (yuvsize == (unsigned long)-1)
+			_throwtj("allocating YUV buffer");
 		if((yuvbuf=(unsigned char *)malloc(yuvsize))==NULL)
 			_throwunix("allocating YUV buffer");
 		memset(yuvbuf, 127, yuvsize);
@@ -237,14 +243,14 @@
 		if(!quiet) printf("Compression error written to %s.\n", tempstr);
 		if(subsamp==TJ_GRAYSCALE)
 		{
-			int index, index2;
+			unsigned long index, index2;
 			for(row=0, index=0; row<h; row++, index+=pitch)
 			{
 				for(col=0, index2=index; col<w; col++, index2+=ps)
 				{
-					int rindex=index2+tjRedOffset[pf];
-					int gindex=index2+tjGreenOffset[pf];
-					int bindex=index2+tjBlueOffset[pf];
+					unsigned long rindex=index2+tjRedOffset[pf];
+					unsigned long gindex=index2+tjGreenOffset[pf];
+					unsigned long bindex=index2+tjBlueOffset[pf];
 					int y=(int)((double)srcbuf[rindex]*0.299
 						+ (double)srcbuf[gindex]*0.587
 						+ (double)srcbuf[bindex]*0.114 + 0.5);
@@ -285,13 +291,16 @@
 	unsigned char **jpegbuf=NULL, *yuvbuf=NULL, *tmpbuf=NULL, *srcptr, *srcptr2;
 	double start, elapsed, elapsedEncode;
 	int totaljpegsize=0, row, col, i, tilew=w, tileh=h, retval=0;
-	int iter, yuvsize=0;
-	unsigned long *jpegsize=NULL;
+	int iter;
+	unsigned long *jpegsize=NULL, yuvsize=0;
 	int ps=tjPixelSize[pf];
 	int ntilesw=1, ntilesh=1, pitch=w*ps;
 	const char *pfStr=pixFormatStr[pf];
 
-	if((tmpbuf=(unsigned char *)malloc(pitch*h)) == NULL)
+	if ((unsigned long long)pitch * (unsigned long long)h >
+	    (unsigned long long)((size_t)-1))
+		_throw("allocating temporary image buffer", "Image is too large");
+	if((tmpbuf=(unsigned char *)malloc((size_t)pitch*h)) == NULL)
 		_throwunix("allocating temporary image buffer");
 
 	if(!quiet)
@@ -317,6 +326,8 @@
 		if((flags&TJFLAG_NOREALLOC)!=0)
 			for(i=0; i<ntilesw*ntilesh; i++)
 			{
+				if (tjBufSize(tilew, tileh, subsamp) > (unsigned long)INT_MAX)
+					_throw("getting buffer size", "Image is too large");
 				if((jpegbuf[i]=(unsigned char *)tjAlloc(tjBufSize(tilew, tileh,
 					subsamp)))==NULL)
 					_throwunix("allocating JPEG tiles");
@@ -334,6 +345,8 @@
 		if(doyuv)
 		{
 			yuvsize=tjBufSizeYUV2(tilew, yuvpad, tileh, subsamp);
+            if (yuvsize == (unsigned long)-1)
+                _throw("allocating YUV buffer", "Image too large");
 			if((yuvbuf=(unsigned char *)malloc(yuvsize))==NULL)
 				_throwunix("allocating YUV buffer");
 			memset(yuvbuf, 127, yuvsize);
@@ -408,7 +421,7 @@
 			{
 				printf("Encode YUV    --> Frame rate:         %f fps\n",
 					(double)iter/elapsedEncode);
-				printf("                  Output image size:  %d bytes\n", yuvsize);
+				printf("                  Output image size:  %lu bytes\n", yuvsize);
 				printf("                  Compression ratio:  %f:1\n",
 					(double)(w*h*ps)/(double)yuvsize);
 				printf("                  Throughput:         %f Megapixels/sec\n",
@@ -453,7 +466,6 @@
 			jpegbuf[i]=NULL;
 		}
 		free(jpegbuf);  jpegbuf=NULL;
-		free(jpegsize);  jpegsize=NULL;
 		if(doyuv)
 		{
 			free(yuvbuf);  yuvbuf=NULL;
@@ -549,9 +561,12 @@
 			_throwunix("allocating JPEG size array");
 		memset(jpegsize, 0, sizeof(unsigned long)*ntilesw*ntilesh);
 
-		if((flags&TJFLAG_NOREALLOC)!=0 || !dotile)
+		if((flags&TJFLAG_NOREALLOC)!=0 &&
+		   (dotile || xformop != TJXOP_NONE || xformopt != 0 || customFilter))
 			for(i=0; i<ntilesw*ntilesh; i++)
 			{
+				if(tjBufSize(tilew, tileh, subsamp) > (unsigned long)INT_MAX)
+					_throw("getting buffer size", "Image is too large");
 				if((jpegbuf[i]=(unsigned char *)tjAlloc(tjBufSize(tilew, tileh,
 					subsamp)))==NULL)
 					_throwunix("allocating JPEG tiles");
@@ -683,7 +698,8 @@
 
 		for(i=0; i<ntilesw*ntilesh; i++)
 		{
-			tjFree(jpegbuf[i]);  jpegbuf[i]=NULL;
+			if (jpegbuf[i]) tjFree(jpegbuf[i]);
+			jpegbuf[i]=NULL;
 		}
 		free(jpegbuf);  jpegbuf=NULL;
 		if(jpegsize) {free(jpegsize);  jpegsize=NULL;}
diff --git a/tjunittest.c b/tjunittest.c
index 6a4022f..e656c1c 100644
--- a/tjunittest.c
+++ b/tjunittest.c
@@ -592,8 +592,42 @@
 	if(dstBuf) tjFree(dstBuf);
 }
 
+#if SIZEOF_SIZE_T == 8
+#define CHECKSIZE(function) { \
+  if ((unsigned long long)size < (unsigned long long)0xFFFFFFFF) \
+    THROW(#function " overflow"); \
+}
+#else
+#define CHECKSIZE(function) { \
+  if (size != (unsigned long)(-1) || \
+      !strcmp(tjGetErrorStr2(NULL), "No error")) \
+    THROW(#function " overflow"); \
+}
+#endif
 
-void bufSizeTest(void)
+static void overflowTest(void)
+{
+  /* Ensure that the various buffer size functions don't overflow */
+  unsigned long size;
+
+  size = tjBufSize(26755, 26755, TJSAMP_444);
+  CHECKSIZE(tjBufSize());
+  size = TJBUFSIZE(26755, 26755);
+  CHECKSIZE(TJBUFSIZE());
+  size = tjBufSizeYUV2(37838, 1, 37838, TJSAMP_444);
+  CHECKSIZE(tjBufSizeYUV2());
+  size = TJBUFSIZEYUV(37838, 37838, TJSAMP_444);
+  CHECKSIZE(TJBUFSIZEYUV());
+  size = tjBufSizeYUV(37838, 37838, TJSAMP_444);
+  CHECKSIZE(tjBufSizeYUV());
+  size = tjPlaneSizeYUV(0, 65536, 0, 65536, TJSAMP_444);
+  CHECKSIZE(tjPlaneSizeYUV());
+
+bailout:
+  return;
+}
+
+static void bufSizeTest(void)
 {
 	int w, h, i, subsamp;
 	unsigned char *srcBuf=NULL, *dstBuf=NULL;
@@ -705,6 +739,7 @@
 	}
 	if(alloc) printf("Testing automatic buffer allocation\n");
 	if(doyuv) num4bf=4;
+	overflowtest();
 	doTest(35, 39, _3byteFormats, 2, TJSAMP_444, "test");
 	doTest(39, 41, _4byteFormats, num4bf, TJSAMP_444, "test");
 	doTest(41, 35, _3byteFormats, 2, TJSAMP_422, "test");
diff --git a/turbojpeg.c b/turbojpeg.c
index 6533b41..cd54ea3 100644
--- a/turbojpeg.c
+++ b/turbojpeg.c
@@ -622,7 +622,7 @@
 DLLEXPORT unsigned long DLLCALL tjBufSize(int width, int height,
 	int jpegSubsamp)
 {
-	unsigned long retval=0;  int mcuw, mcuh, chromasf;
+	unsigned long long retval=0;  int mcuw, mcuh, chromasf;
 	if(width<1 || height<1 || jpegSubsamp<0 || jpegSubsamp>=NUMSUBOPT)
 		_throw("tjBufSize(): Invalid argument");
 
@@ -632,15 +632,17 @@
 	mcuw=tjMCUWidth[jpegSubsamp];
 	mcuh=tjMCUHeight[jpegSubsamp];
 	chromasf=jpegSubsamp==TJSAMP_GRAY? 0: 4*64/(mcuw*mcuh);
-	retval=PAD(width, mcuw) * PAD(height, mcuh) * (2 + chromasf) + 2048;
+	retval=PAD(width, mcuw) * PAD(height, mcuh) * (2ULL + chromasf) + 2048ULL;
+	if (retval > (unsigned long long)((unsigned long)-1))
+		_throw("tjBufSize(): Image is too large");
 
 	bailout:
-	return retval;
+	return (unsigned long)retval;
 }
 
 DLLEXPORT unsigned long DLLCALL TJBUFSIZE(int width, int height)
 {
-	unsigned long retval=0;
+	unsigned long long retval=0;
 	if(width<1 || height<1)
 		_throw("TJBUFSIZE(): Invalid argument");
 
@@ -648,16 +650,19 @@
 	   larger than the uncompressed input (we wouldn't mention it if it hadn't
 	   happened before.) */
 	retval=PAD(width, 16) * PAD(height, 16) * 6 + 2048;
+	if (retval > (unsigned long long)((unsigned long)-1))
+		_throw("tjBufSize(): Image is too large");
 
 	bailout:
-	return retval;
+	return (unsigned long)retval;
 }
 
 
 DLLEXPORT unsigned long DLLCALL tjBufSizeYUV2(int width, int pad, int height,
 	int subsamp)
 {
-	int retval=0, nc, i;
+	unsigned long long retval=0;
+	int nc, i;
 
 	if(subsamp<0 || subsamp>=NUMSUBOPT)
 		_throw("tjBufSizeYUV2(): Invalid argument");
@@ -669,11 +674,13 @@
 		int stride=PAD(pw, pad);
 		int ph=tjPlaneHeight(i, height, subsamp);
 		if(pw<0 || ph<0) return -1;
-		else retval+=stride*ph;
+		else retval+=(unsigned long long)stride*ph;
 	}
+	if (retval > (unsigned long long)((unsigned long)-1))
+		_throw("tjBufSize(): Image is too large");
 
 	bailout:
-	return retval;
+	return (unsigned long)retval;
 }
 
 DLLEXPORT unsigned long DLLCALL tjBufSizeYUV(int width, int height,
@@ -734,7 +741,7 @@
 DLLEXPORT unsigned long DLLCALL tjPlaneSizeYUV(int componentID, int width,
 	int stride, int height, int subsamp)
 {
-	unsigned long retval=0;
+	unsigned long long retval=0;
 	int pw, ph;
 
 	if(width<1 || height<1 || subsamp<0 || subsamp>=NUMSUBOPT)
@@ -747,10 +754,12 @@
 	if(stride==0) stride=pw;
 	else stride=abs(stride);
 
-	retval=stride*(ph-1)+pw;
+	retval=(unsigned long long)stride*(ph-1)+pw;
+	if (retval > (unsigned long long)((unsigned long)-1))
+		_throw("tjPlaneSizeYUV(): Image is too large");
 
 	bailout:
-	return retval;
+	return (unsigned long)retval;
 }
 
 
@@ -812,8 +821,8 @@
 	for(i=0; i<height; i++)
 	{
 		if(flags&TJFLAG_BOTTOMUP)
-			row_pointer[i]=(JSAMPROW)&srcBuf[(height-i-1)*pitch];
-		else row_pointer[i]=(JSAMPROW)&srcBuf[i*pitch];
+			row_pointer[i]=(JSAMPROW)&srcBuf[(height-i-1)*(size_t)pitch];
+		else row_pointer[i]=(JSAMPROW)&srcBuf[i*(size_t)pitch];
 	}
 	while(cinfo->next_scanline<cinfo->image_height)
 	{
@@ -939,8 +948,8 @@
 	for(i=0; i<height; i++)
 	{
 		if(flags&TJFLAG_BOTTOMUP)
-			row_pointer[i]=(JSAMPROW)&srcBuf[(height-i-1)*pitch];
-		else row_pointer[i]=(JSAMPROW)&srcBuf[i*pitch];
+			row_pointer[i]=(JSAMPROW)&srcBuf[(height-i-1)*(size_t)pitch];
+		else row_pointer[i]=(JSAMPROW)&srcBuf[i*(size_t)pitch];
 	}
 	if(height<ph0)
 		for(i=height; i<ph0; i++) row_pointer[i]=row_pointer[height-1];
@@ -1441,8 +1450,8 @@
 	for(i=0; i<(int)dinfo->output_height; i++)
 	{
 		if(flags&TJFLAG_BOTTOMUP)
-			row_pointer[i]=&dstBuf[(dinfo->output_height-i-1)*pitch];
-		else row_pointer[i]=&dstBuf[i*pitch];
+			row_pointer[i]=&dstBuf[(dinfo->output_height-i-1)*(size_t)pitch];
+		else row_pointer[i]=&dstBuf[i*(size_t)pitch];
 	}
 	while(dinfo->output_scanline<dinfo->output_height)
 	{
@@ -1627,8 +1636,8 @@
 		_throw("tjDecodeYUVPlanes(): Memory allocation failure");
 	for(i=0; i<height; i++)
 	{
-		if(flags&TJFLAG_BOTTOMUP) row_pointer[i]=&dstBuf[(height-i-1)*pitch];
-		else row_pointer[i]=&dstBuf[i*pitch];
+		if(flags&TJFLAG_BOTTOMUP) row_pointer[i]=&dstBuf[(height-i-1)*(size_t)pitch];
+		else row_pointer[i]=&dstBuf[i*(size_t)pitch];
 	}
 	if(height<ph0)
 		for(i=height; i<ph0; i++) row_pointer[i]=row_pointer[height-1];