trusty: cast-auth: fix incorrect return value on error path
The function returns a bool to identify the keytype, but would return an
error code on memory failure. Change the function to avoid the memory
allocation failure path.
Bug: 332371854
Test: build.py qemu-generic-arm64-test-debug --test com.android.trusty.cast_auth.test
Change-Id: Idc0f4b2484ec5b00bf97eed3647196ef2da463bb
diff --git a/app/cast_auth_impl.cc b/app/cast_auth_impl.cc
index cfa8a0f..b1ccdb4 100644
--- a/app/cast_auth_impl.cc
+++ b/app/cast_auth_impl.cc
@@ -45,13 +45,10 @@
using android::binder::Status;
bool is_plaintext_rsa_2048_private_key(const std::vector<uint8_t>& key) {
- bssl::UniquePtr<BIO> bio(BIO_new_mem_buf(key.data(), key.size()));
- if (!bio) {
- TLOGE("is_plaintext_rsa_2048_private_key: failed to allocate memory for the "
- "device key\n");
- return ERR_NO_MEMORY;
- }
- bssl::UniquePtr<RSA> rsa(d2i_RSAPrivateKey_bio(bio.get(), NULL));
+ const uint8_t* key_data = key.data();
+
+ bssl::UniquePtr<RSA> rsa(d2i_RSAPrivateKey(NULL, &key_data, key.size()));
+
return rsa && RSA_size(rsa.get()) == RSA_2048_SIZE_BYTES;
}
