8203182: Release session if initialization of SunPKCS11 Signature fails

Ensure session is properly released in P11Signature class

Reviewed-by: valeriep
diff --git a/src/jdk.crypto.cryptoki/share/classes/sun/security/pkcs11/P11Signature.java b/src/jdk.crypto.cryptoki/share/classes/sun/security/pkcs11/P11Signature.java
index 9c00ec9..ebb5416 100644
--- a/src/jdk.crypto.cryptoki/share/classes/sun/security/pkcs11/P11Signature.java
+++ b/src/jdk.crypto.cryptoki/share/classes/sun/security/pkcs11/P11Signature.java
@@ -283,47 +283,51 @@
             session = token.killSession(session);
             return;
         }
-        // "cancel" operation by finishing it
-        // XXX make sure all this always works correctly
-        if (mode == M_SIGN) {
-            try {
-                if (type == T_UPDATE) {
-                    token.p11.C_SignFinal(session.id(), 0);
-                } else {
-                    byte[] digest;
-                    if (type == T_DIGEST) {
-                        digest = md.digest();
-                    } else { // T_RAW
-                        digest = buffer;
+        try {
+            // "cancel" operation by finishing it
+            // XXX make sure all this always works correctly
+            if (mode == M_SIGN) {
+                try {
+                    if (type == T_UPDATE) {
+                        token.p11.C_SignFinal(session.id(), 0);
+                    } else {
+                        byte[] digest;
+                        if (type == T_DIGEST) {
+                            digest = md.digest();
+                        } else { // T_RAW
+                            digest = buffer;
+                        }
+                        token.p11.C_Sign(session.id(), digest);
                     }
-                    token.p11.C_Sign(session.id(), digest);
+                } catch (PKCS11Exception e) {
+                    throw new ProviderException("cancel failed", e);
                 }
-            } catch (PKCS11Exception e) {
-                throw new ProviderException("cancel failed", e);
-            }
-        } else { // M_VERIFY
-            try {
-                byte[] signature;
-                if (keyAlgorithm.equals("DSA")) {
-                    signature = new byte[40];
-                } else {
-                    signature = new byte[(p11Key.length() + 7) >> 3];
-                }
-                if (type == T_UPDATE) {
-                    token.p11.C_VerifyFinal(session.id(), signature);
-                } else {
-                    byte[] digest;
-                    if (type == T_DIGEST) {
-                        digest = md.digest();
-                    } else { // T_RAW
-                        digest = buffer;
+            } else { // M_VERIFY
+                try {
+                    byte[] signature;
+                    if (keyAlgorithm.equals("DSA")) {
+                        signature = new byte[40];
+                    } else {
+                        signature = new byte[(p11Key.length() + 7) >> 3];
                     }
-                    token.p11.C_Verify(session.id(), digest, signature);
+                    if (type == T_UPDATE) {
+                        token.p11.C_VerifyFinal(session.id(), signature);
+                    } else {
+                        byte[] digest;
+                        if (type == T_DIGEST) {
+                            digest = md.digest();
+                        } else { // T_RAW
+                            digest = buffer;
+                        }
+                        token.p11.C_Verify(session.id(), digest, signature);
+                    }
+                } catch (PKCS11Exception e) {
+                    // will fail since the signature is incorrect
+                    // XXX check error code
                 }
-            } catch (PKCS11Exception e) {
-                // will fail since the signature is incorrect
-                // XXX check error code
             }
+        } finally {
+            session = token.releaseSession(session);
         }
     }
 
@@ -342,6 +346,8 @@
             }
             initialized = true;
         } catch (PKCS11Exception e) {
+            // release session when initialization failed
+            session = token.releaseSession(session);
             throw new ProviderException("Initialization failed", e);
         }
         if (bytesProcessed != 0) {
@@ -511,6 +517,8 @@
                 }
                 bytesProcessed += len;
             } catch (PKCS11Exception e) {
+                initialized = false;
+                session = token.releaseSession(session);
                 throw new ProviderException(e);
             }
             break;
@@ -559,6 +567,8 @@
                 bytesProcessed += len;
                 byteBuffer.position(ofs + len);
             } catch (PKCS11Exception e) {
+                initialized = false;
+                session = token.releaseSession(session);
                 throw new ProviderException("Update failed", e);
             }
             break;