8130875: Ucrypto library leaks memory when null output buffer is specified
Avoid null output buffer to work around Solaris memory leak bug in Ucrypto library
Reviewed-by: ascarpino
diff --git a/jdk/src/jdk.crypto.ucrypto/solaris/native/libj2ucrypto/nativeCrypto.c b/jdk/src/jdk.crypto.ucrypto/solaris/native/libj2ucrypto/nativeCrypto.c
index a65eaca..e4d559e 100644
--- a/jdk/src/jdk.crypto.ucrypto/solaris/native/libj2ucrypto/nativeCrypto.c
+++ b/jdk/src/jdk.crypto.ucrypto/solaris/native/libj2ucrypto/nativeCrypto.c
@@ -435,6 +435,11 @@
int rv = 0;
context = (crypto_ctx_t *) pContext;
+ // Avoid null output buffer to workaround Solaris bug21481818 (fixed in S12)
+ if (bufOut == NULL) {
+ bufOut = (unsigned char *)(&outLen);
+ outLen = 0;
+ }
rv = CipherFinal(context, encrypt, (unsigned char*)bufOut, outOfs, &outLen);
free(context);
if (rv) {
@@ -648,7 +653,8 @@
// out is null when nativeFinal() is called solely for resource clean up
if (out == NULL) {
- bufOut = NULL;
+ // Avoid null output buffer to workaround Solaris bug21481818 (fixed in S12)
+ bufOut = (unsigned char *)(&outLen);
outLen = 0;
} else {
outLen = (*env)->GetArrayLength(env, out) - outOfs;
@@ -661,10 +667,12 @@
rv = CipherFinal(context, encrypt, bufOut, 0, &outLen);
if (rv) {
free(context);
- free(bufOut);
+ if (outLen != 0) {
+ free(bufOut);
+ }
return -rv;
} else {
- if (bufOut != NULL) {
+ if (bufOut != NULL && outLen != 0) {
(*env)->SetByteArrayRegion(env, out, outOfs, outLen, (jbyte *)bufOut);
free(bufOut);
}
