Return correct error from keymaster0engine for large RSA input

Also, ensure that we always put some error on the OpenSSL error queue
whenever a wrapped keymaster0 operation fails.  Higher layers will look
a the last entry on the queue and use it to determine what error code to
return.  Not putting any error on the queue means that those higher
layers will get whatever error was last enqueued, making the result
effectively random.  Non-determinism bad.

(cherry-picked from commit 22d2355b7edc470949c163e47ba8e837a1a87f47)

Bug: 25337630
Change-Id: I701ab735dd089f5258b2252f543906d9f3baa7a2
3 files changed
tree: e853546fbe63e8e96df47be8d12fc2fea5c7c99a
  1. .clang-format
  2. .gitignore
  3. Android.mk
  4. List.h
  5. Makefile
  6. ae.h
  7. aes_key.cpp
  8. aes_key.h
  9. aes_operation.cpp
  10. aes_operation.h
  11. android_keymaster.cpp
  12. android_keymaster_messages.cpp
  13. android_keymaster_messages_test.cpp
  14. android_keymaster_test.cpp
  15. android_keymaster_test_utils.cpp
  16. android_keymaster_test_utils.h
  17. android_keymaster_utils.cpp
  18. asymmetric_key.cpp
  19. asymmetric_key.h
  20. asymmetric_key_factory.cpp
  21. auth_encrypted_key_blob.cpp
  22. auth_encrypted_key_blob.h
  23. authorization_set.cpp
  24. authorization_set_test.cpp
  25. ec_key.cpp
  26. ec_key.h
  27. ec_key_factory.cpp
  28. ec_keymaster0_key.cpp
  29. ec_keymaster0_key.h
  30. ec_keymaster1_key.cpp
  31. ec_keymaster1_key.h
  32. ec_privkey_pk8.der
  33. ecdsa_keymaster1_operation.cpp
  34. ecdsa_keymaster1_operation.h
  35. ecdsa_operation.cpp
  36. ecdsa_operation.h
  37. gtest_main.cpp
  38. hkdf.cpp
  39. hkdf.h
  40. hkdf_test.cpp
  41. hmac.cpp
  42. hmac.h
  43. hmac_key.cpp
  44. hmac_key.h
  45. hmac_operation.cpp
  46. hmac_operation.h
  47. hmac_test.cpp
  48. include/
  49. integrity_assured_key_blob.cpp
  50. integrity_assured_key_blob.h
  51. key.cpp
  52. key.h
  53. key_blob_test.cpp
  54. keymaster0_engine.cpp
  55. keymaster0_engine.h
  56. keymaster1_engine.cpp
  57. keymaster1_engine.h
  58. keymaster_enforcement.cpp
  59. keymaster_enforcement_test.cpp
  60. km0_sw_rsa_512.blob
  61. km1_sw_ecdsa_256.blob
  62. km1_sw_rsa_512.blob
  63. km1_sw_rsa_512_unversioned.blob
  64. logger.cpp
  65. ocb.c
  66. ocb_utils.cpp
  67. ocb_utils.h
  68. openssl_err.cpp
  69. openssl_err.h
  70. openssl_utils.cpp
  71. openssl_utils.h
  72. operation.cpp
  73. operation.h
  74. operation_table.cpp
  75. operation_table.h
  76. rsa_key.cpp
  77. rsa_key.h
  78. rsa_key_factory.cpp
  79. rsa_keymaster0_key.cpp
  80. rsa_keymaster0_key.h
  81. rsa_keymaster1_key.cpp
  82. rsa_keymaster1_key.h
  83. rsa_keymaster1_operation.cpp
  84. rsa_keymaster1_operation.h
  85. rsa_operation.cpp
  86. rsa_operation.h
  87. rsa_privkey_pk8.der
  88. serializable.cpp
  89. soft_keymaster_context.cpp
  90. soft_keymaster_device.cpp
  91. soft_keymaster_logger.cpp
  92. symmetric_key.cpp
  93. symmetric_key.h
  94. valgrind.supp