8158356: SIGSEGV when attempting to rotate BufferedImage using AffineTransform by NaN degrees
Reviewed-by: prr, flar
diff --git a/jdk/src/share/native/sun/awt/medialib/awt_ImagingLib.c b/jdk/src/share/native/sun/awt/medialib/awt_ImagingLib.c
index 472b5ee..2845899 100644
--- a/jdk/src/share/native/sun/awt/medialib/awt_ImagingLib.c
+++ b/jdk/src/share/native/sun/awt/medialib/awt_ImagingLib.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2014, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2016, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -772,6 +772,7 @@
mlib_image *src;
mlib_image *dst;
int i;
+ int j = 0;
int retStatus = 1;
mlib_status status;
double *matrix;
@@ -824,6 +825,15 @@
return 0;
}
+ /* Check for invalid double value in transformation matrix */
+ for (j = 0; j < 6; j++) {
+
+ if (!(IS_FINITE(matrix[j]))) {
+ (*env)->ReleasePrimitiveArrayCritical(env, jmatrix, matrix, JNI_ABORT);
+ return 0;
+ }
+ }
+
if (s_printIt) {
printf("matrix is %g %g %g %g %g %g\n", matrix[0], matrix[1],
matrix[2], matrix[3], matrix[4], matrix[5]);
@@ -980,6 +990,7 @@
mlib_image *src;
mlib_image *dst;
int i;
+ int j = 0;
int retStatus = 1;
mlib_status status;
double *matrix;
@@ -1044,6 +1055,17 @@
return 0;
}
+ /* Check for invalid double value in transformation matrix */
+ for (j = 0; j < 6; j++) {
+
+ if (!(IS_FINITE(matrix[j]))) {
+ (*env)->ReleasePrimitiveArrayCritical(env, jmatrix, matrix, JNI_ABORT);
+ free(srcRasterP);
+ free(dstRasterP);
+ return 0;
+ }
+ }
+
if (s_printIt) {
printf("matrix is %g %g %g %g %g %g\n", matrix[0], matrix[1],
matrix[2], matrix[3], matrix[4], matrix[5]);
diff --git a/jdk/src/share/native/sun/awt/medialib/mlib_ImageScanPoly.c b/jdk/src/share/native/sun/awt/medialib/mlib_ImageScanPoly.c
index 34c2171..ccc1572 100644
--- a/jdk/src/share/native/sun/awt/medialib/mlib_ImageScanPoly.c
+++ b/jdk/src/share/native/sun/awt/medialib/mlib_ImageScanPoly.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2003, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2016, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -33,6 +33,8 @@
#include "mlib_image.h"
#include "mlib_SysMath.h"
#include "mlib_ImageAffine.h"
+#include "safe_math.h"
+
/***************************************************************/
mlib_status mlib_AffineEdges(mlib_affine_param *param,
@@ -83,6 +85,12 @@
dstYStride = mlib_ImageGetStride(dst);
paddings = mlib_ImageGetPaddings(src);
+ /* All the transformation matrix parameters should be finite. if not, return failure */
+ if (!(IS_FINITE(a) && IS_FINITE(b) && IS_FINITE(c) && IS_FINITE(d) &&
+ IS_FINITE(tx) && IS_FINITE(ty))) {
+ return MLIB_FAILURE;
+ }
+
if (srcWidth >= (1 << 15) || srcHeight >= (1 << 15)) {
return MLIB_FAILURE;
}
@@ -288,6 +296,10 @@
if (dY1 == dY2)
continue;
+ if (!(IS_FINITE(slope))) {
+ continue;
+ }
+
if (dY1 < 0.0)
y1 = 0;
else {
@@ -328,6 +340,10 @@
if (dY1 == dY2)
continue;
+ if (!(IS_FINITE(slope))) {
+ continue;
+ }
+
if (dY1 < 0.0)
y1 = 0;
else {
diff --git a/jdk/src/share/native/sun/awt/medialib/safe_math.h b/jdk/src/share/native/sun/awt/medialib/safe_math.h
index 34c1fc5..a87eba1 100644
--- a/jdk/src/share/native/sun/awt/medialib/safe_math.h
+++ b/jdk/src/share/native/sun/awt/medialib/safe_math.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -26,10 +26,15 @@
#ifndef __SAFE_MATH_H__
#define __SAFE_MATH_H__
+#include "mlib_types.h"
+
#define SAFE_TO_MULT(a, b) \
(((a) > 0) && ((b) >= 0) && ((0x7fffffff / (a)) > (b)))
#define SAFE_TO_ADD(a, b) \
(((a) >= 0) && ((b) >= 0) && ((0x7fffffff - (a)) > (b)))
+#define IS_FINITE(a) \
+ (((a) >= MLIB_D64_MIN) && ((a) <= MLIB_D64_MAX))
+
#endif // __SAFE_MATH_H__
diff --git a/jdk/test/java/awt/geom/AffineTransform/InvalidTransformParameterTest.java b/jdk/test/java/awt/geom/AffineTransform/InvalidTransformParameterTest.java
new file mode 100644
index 0000000..f8c36ec
--- /dev/null
+++ b/jdk/test/java/awt/geom/AffineTransform/InvalidTransformParameterTest.java
@@ -0,0 +1,138 @@
+/*
+ * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/* @test
+ * @bug 8158356
+ * @summary Test AffineTransform transformations do not result in SIGSEGV
+ * if NaN or infinity parameter is passed as argument.
+ * @run main InvalidTransformParameterTest
+ */
+
+import java.awt.geom.AffineTransform;
+import java.awt.image.AffineTransformOp;
+import java.awt.image.BufferedImage;
+import java.awt.image.ImagingOpException;
+import java.awt.Point;
+import java.awt.image.DataBuffer;
+import java.awt.image.DataBufferByte;
+import java.awt.image.PixelInterleavedSampleModel;
+import java.awt.image.Raster;
+import java.awt.image.WritableRaster;
+import java.awt.image.RasterOp;
+import java.awt.image.SampleModel;
+
+public class InvalidTransformParameterTest {
+
+ public static void main(String[] args) {
+ int count = 0;
+ final int testScenarios = 12;
+ double NaNArg = 0.0 / 0.0;
+ double positiveInfArg = 1.0 / 0.0;
+ double negativeInfArg = -1.0 / 0.0;
+
+ BufferedImage img = new BufferedImage(5, 5, BufferedImage.TYPE_INT_ARGB);
+
+ AffineTransform[] inputTransforms = new AffineTransform[testScenarios];
+
+ for (int i = 0; i < inputTransforms.length; i++) {
+ inputTransforms[i] = new AffineTransform();
+ }
+
+ inputTransforms[0].rotate(NaNArg, img.getWidth()/2, img.getHeight()/2);
+ inputTransforms[1].translate(NaNArg, NaNArg);
+ inputTransforms[2].scale(NaNArg, NaNArg);
+ inputTransforms[3].shear(NaNArg, NaNArg);
+
+ inputTransforms[4].rotate(positiveInfArg, img.getWidth()/2, img.getHeight()/2);
+ inputTransforms[5].translate(positiveInfArg, positiveInfArg);
+ inputTransforms[6].scale(positiveInfArg, positiveInfArg);
+ inputTransforms[7].shear(positiveInfArg, positiveInfArg);
+
+ inputTransforms[8].rotate(negativeInfArg, img.getWidth()/2, img.getHeight()/2);
+ inputTransforms[9].translate(negativeInfArg, negativeInfArg);
+ inputTransforms[10].scale(negativeInfArg, negativeInfArg);
+ inputTransforms[11].shear(negativeInfArg, negativeInfArg);
+
+ // Test BufferedImage AffineTransform ---------------------------------
+
+ for (int i = 0; i < inputTransforms.length; i++) {
+ try {
+ testImageTransform(img, inputTransforms[i]);
+ } catch (ImagingOpException ex) {
+ count++;
+ }
+ }
+
+ if (count != testScenarios) {
+ throw new RuntimeException("Test failed. All test scenarios did not"
+ + " result in exception as expected.");
+ }
+
+ // Test Raster AffineTransform ---------------------------------
+
+ count = 0;
+ int[] bandOffsets = {0};
+ Point location = new Point(0, 0);
+ DataBuffer db = new DataBufferByte(10 * 10);
+ SampleModel sm = new PixelInterleavedSampleModel(DataBuffer.TYPE_BYTE,
+ 10, 10, 1, 10,
+ bandOffsets);
+
+ Raster src = Raster.createRaster(sm, db, location);
+ WritableRaster dst = Raster.createWritableRaster(sm, db, location);
+
+ for (int i = 0; i < inputTransforms.length; i++) {
+ try {
+ testRasterTransform(src, dst, inputTransforms[i]);
+ } catch (ImagingOpException ex) {
+ count++;
+ }
+ }
+
+ if (count != testScenarios) {
+ throw new RuntimeException("Test failed. All test scenarios did not"
+ + " result in exception as expected.");
+ }
+ }
+
+ public static BufferedImage testImageTransform(BufferedImage image,
+ AffineTransform transform) {
+ AffineTransformOp op =
+ new AffineTransformOp(transform, AffineTransformOp.TYPE_BILINEAR);
+
+ BufferedImage transformedImage = new BufferedImage(image.getWidth(),
+ image.getHeight(),
+ image.getType());
+
+ return op.filter(image, transformedImage);
+ }
+
+ public static Raster testRasterTransform(Raster src, WritableRaster dst,
+ AffineTransform transform) {
+ AffineTransformOp op =
+ new AffineTransformOp(transform, AffineTransformOp.TYPE_BILINEAR);
+
+ return op.filter(src, dst);
+ }
+}
+
