Java: Guard against int overflow in size methods

Because Java array sizes are ints, the various size methods in the TJ
class have int return values.  Thus, we have to guard against signed
int overflow at the JNI level, because the C functions can return sizes
greater than INT_MAX.

This also adds a test for TJ.planeWidth() and TJ.planeHeight(), in order
to validate 8a1526a in Java.

Author: dcommander

java/TJUnitTest.java

@@ -844,34 +844,46 @@ static void overflowTest() throws Exception {
 
     try {
       exception = false;
-      size = TJ.bufSize(26755, 26755, TJ.SAMP_444);
+      size = TJ.bufSize(18919, 18919, TJ.SAMP_444);
     } catch (Exception e) { exception = true; }
     if (!exception || size != 0)
       throw new Exception("TJ.bufSize() overflow");
     try {
       exception = false;
-      size = TJ.bufSizeYUV(37838, 1, 37838, TJ.SAMP_444);
+      size = TJ.bufSizeYUV(26755, 1, 26755, TJ.SAMP_444);
     } catch (Exception e) { exception = true; }
     if (!exception || size != 0)
       throw new Exception("TJ.bufSizeYUV() overflow");
     try {
       exception = false;
-      size = TJ.bufSizeYUV(37837, 3, 37837, TJ.SAMP_444);
+      size = TJ.bufSizeYUV(26754, 3, 26754, TJ.SAMP_444);
     } catch (Exception e) { exception = true; }
     if (!exception || size != 0)
       throw new Exception("TJ.bufSizeYUV() overflow");
     try {
       exception = false;
-      size = TJ.bufSizeYUV(37837, -1, 37837, TJ.SAMP_444);
+      size = TJ.bufSizeYUV(26754, -1, 26754, TJ.SAMP_444);
     } catch (Exception e) { exception = true; }
     if (!exception || size != 0)
       throw new Exception("TJ.bufSizeYUV() overflow");
     try {
       exception = false;
-      size = TJ.planeSizeYUV(0, 65536, 0, 65536, TJ.SAMP_444);
+      size = TJ.planeSizeYUV(0, 46341, 0, 46341, TJ.SAMP_444);
     } catch (Exception e) { exception = true; }
     if (!exception || size != 0)
       throw new Exception("TJ.planeSizeYUV() overflow");
+    try {
+      exception = false;
+      size = TJ.planeWidth(0, Integer.MAX_VALUE, TJ.SAMP_420);
+    } catch (Exception e) { exception = true; }
+    if (!exception || size != 0)
+      throw new Exception("TJ.planeWidth() overflow");
+    try {
+      exception = false;
+      size = TJ.planeHeight(0, Integer.MAX_VALUE, TJ.SAMP_420);
+    } catch (Exception e) { exception = true; }
+    if (!exception || size != 0)
+      throw new Exception("TJ.planeHeight() overflow");
   }
 
   static void bufSizeTest() throws Exception {

turbojpeg-jni.c

@@ -26,6 +26,7 @@
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include <limits.h>
 #include "turbojpeg.h"
 #include "jinclude.h"
 #include <jni.h>
@@ -135,7 +136,7 @@ JNIEXPORT jint JNICALL Java_org_libjpegturbo_turbojpeg_TJ_bufSize
   unsigned long retval = tjBufSize(width, height, jpegSubsamp);
 
   if (retval == (unsigned long)-1) THROW_ARG(tjGetErrorStr());
-  if (retval > (unsigned long)((unsigned int)-1))
+  if (retval > (unsigned long)INT_MAX)
     THROW_ARG("Image is too large");
 
 bailout:
@@ -149,7 +150,7 @@ JNIEXPORT jint JNICALL Java_org_libjpegturbo_turbojpeg_TJ_bufSizeYUV__IIII
   unsigned long retval = tjBufSizeYUV2(width, align, height, subsamp);
 
   if (retval == (unsigned long)-1) THROW_ARG(tjGetErrorStr());
-  if (retval > (unsigned long)((unsigned int)-1))
+  if (retval > (unsigned long)INT_MAX)
     THROW_ARG("Image is too large");
 
 bailout:
@@ -174,7 +175,7 @@ JNIEXPORT jint JNICALL Java_org_libjpegturbo_turbojpeg_TJ_planeSizeYUV__IIIII
                                         subsamp);
 
   if (retval == (unsigned long)-1) THROW_ARG(tjGetErrorStr());
-  if (retval > (unsigned long)((unsigned int)-1))
+  if (retval > (unsigned long)INT_MAX)
     THROW_ARG("Image is too large");
 
 bailout: