lib: sm: Implement VM availability messages Handle framework VM creation and destruction messages. We basically only care about the latter, and handle them by managing a list of notifiers that clients can register to be notified of VM destruction events. Bug: 280886201 Change-Id: I517a18882cd2733a4c28b41e2bf113a815765e3a
diff --git a/lib/sm/include/lib/sm.h b/lib/sm/include/lib/sm.h index 5c96b3f..6e74a4a 100644 --- a/lib/sm/include/lib/sm.h +++ b/lib/sm/include/lib/sm.h
@@ -25,9 +25,11 @@ #include <lib/extmem/extmem.h> #include <lib/sm/smcall.h> +#include <lk/list.h> #include <stdbool.h> #include <stddef.h> #include <sys/types.h> +#include <uapi/err.h> #define PRIxNS_ADDR PRIx64 @@ -111,4 +113,82 @@ ns_addr_t* ppa, uint* pmmu); +/** + * struct sm_vm_notifier - VM notifier to call on VM events. + * @node: List node in notifiers list. + * @client_id: VM identifier. + * @destroy: Destruction event callback. + * + * The &struct sm_vm_notifier object must exist at least until both the + * @destroy callback is called or sm_vm_notifier_unregister() has returned. + * If sm_vm_notifier_unregister() has returned, the callback will not be called + * and it is safe to free the object. + * + * If the client intends to call sm_vm_notifier_unregister() at any point + * after the callback returns, it should keep the notifier object alive + * until after the last call to sm_vm_notifier_unregister(). + * Note that all such invocations will just return %ERR_NOT_FOUND. + */ +struct sm_vm_notifier { + struct list_node node; + ext_mem_obj_id_t client_id; + status_t (*destroy)(struct sm_vm_notifier*); +}; + +/** + * sm_vm_notifier_init() - Initialize a notifier. + * @notif: Pointer to notifier + * @client_id: VM identifier to set in notifier. + * @destroy: Destruction callback. + */ +static inline status_t sm_vm_notifier_init( + struct sm_vm_notifier* notif, + ext_mem_obj_id_t client_id, + status_t (*destroy)(struct sm_vm_notifier*)) { + if (!notif) { + return ERR_INVALID_ARGS; + } + if (!destroy) { + return ERR_INVALID_ARGS; + } + + list_clear_node(¬if->node); + notif->client_id = client_id; + notif->destroy = destroy; + + return NO_ERROR; +} + +/** + * sm_vm_notifier_register() - Register a notifier for VM events. + * @notif: Pointer to notifier. + * + * Return: + * * %0 in case of success + * * %ERR_INVALID_ARGS if @notif is invalid + * * %ERR_NOT_FOUND if the VM has not been created + * * %ERR_BAD_STATE if the VM is already present and in an invalid state + * + * The contents of @notif should be initialized using + * sm_vm_notifier_init(). + * + * The function does not take ownership of @notif. + */ +status_t sm_vm_notifier_register(struct sm_vm_notifier* notif); + +/** + * sm_vm_notifier_unregister() - Unregister a notifier for VM events. + * @notif: Pointer to notifier. + * + * Return: + * * %0 in case of success + * * %ERR_INVALID_ARGS if @notif is invalid + * * %ERR_NOT_FOUND if the notifier has not been previously registered + * + * The function will block until the destruction callback finishes + * if it is already running on another thread. If the destruction callback + * is not already running, it will not be called. + */ +status_t sm_vm_notifier_unregister(struct sm_vm_notifier* notif); + #endif /* __SM_H */
diff --git a/lib/sm/sm.c b/lib/sm/sm.c index c51f203..db46817 100644 --- a/lib/sm/sm.c +++ b/lib/sm/sm.c
@@ -29,6 +29,7 @@ #include <kernel/thread.h> #include <kernel/vm.h> #include <lib/arm_ffa/arm_ffa.h> +#include <lib/binary_search_tree.h> #include <lib/heap.h> #include <lib/sm.h> #include <lib/sm/sm_err.h> @@ -70,6 +71,52 @@ #if LIB_SM_WITH_FFA_LOOP static bool sm_use_ffa = true; static atomic_bool sm_ffa_valid_call; + +enum sm_vm_state { + SM_VM_STATE_FRESH, + SM_VM_STATE_AVAILABLE, + SM_VM_STATE_DESTROY_NOTIFYING, + SM_VM_STATE_DESTROY_NOTIFIED, + SM_VM_STATE_READY_TO_FREE +}; + +struct sm_vm { + struct bst_node node; + enum sm_vm_state state; + ext_mem_obj_id_t client_id; + struct list_node notifiers; +}; + +/* + * VM ID to create; can be one of two values: + * * Non-negative 16-bit VM ID, or + * * -1 when no VM needs to be created + */ +static int32_t sm_vm_to_create = -1; +static struct bst_root sm_vm_tree = BST_ROOT_INITIAL_VALUE; +static struct bst_root sm_vm_free_tree = BST_ROOT_INITIAL_VALUE; +static spin_lock_t sm_vm_lock; +static event_t sm_vm_event = + EVENT_INITIAL_VALUE(sm_vm_event, 0, EVENT_FLAG_AUTOUNSIGNAL); +static thread_t* sm_vm_notifier_thread; +static atomic_uintptr_t sm_vm_active_notifier; +static event_t sm_vm_notifier_done_event = + EVENT_INITIAL_VALUE(sm_vm_notifier_done_event, + 0, + EVENT_FLAG_AUTOUNSIGNAL); + +/* + * Placeholder compatibility VM for environments without hypervisors + * and for the bootloader that may call Trusty before the hypervisor has + * initialized. This pseudo-VM does not get creation or destruction messages + * so we add and remove it from the tree manually. + */ +static struct sm_vm sm_vm_compat0 = { + .node = BST_NODE_INITIAL_VALUE, + .state = SM_VM_STATE_FRESH, + .client_id = 0, + .notifiers = LIST_INITIAL_VALUE(sm_vm_compat0.notifiers), +}; #else static bool sm_use_ffa = false; #endif @@ -318,6 +365,372 @@ } } +static int sm_vm_compare_key(const struct bst_node* a, const void* b) { + const struct sm_vm* vm = containerof(a, struct sm_vm, node); + ext_mem_obj_id_t key = *(ext_mem_obj_id_t*)b; + + if (key > vm->client_id) { + return 1; + } else if (key < vm->client_id) { + return -1; + } else { + return 0; + } +} + +static int sm_vm_compare(struct bst_node* a, struct bst_node* b) { + const struct sm_vm* vm_b = containerof(b, struct sm_vm, node); + + return sm_vm_compare_key(a, &vm_b->client_id); +} + +static void sm_vm_add_compat0_locked(void) { + DEBUG_ASSERT(spin_lock_held(&sm_vm_lock)); + + if (!bst_is_empty(&sm_vm_tree)) { + /* + * There is already a VM in the tree, so we don't need + * to add the compatibility VM 0 explicitly. + */ + return; + } + if (sm_vm_to_create != -1) { + /* The tree is empty but we have a pending VM queued up for creation */ + return; + } + + DEBUG_ASSERT(sm_vm_compat0.state == SM_VM_STATE_FRESH || + sm_vm_compat0.state == SM_VM_STATE_READY_TO_FREE); + if (!bst_insert(&sm_vm_tree, &sm_vm_compat0.node, sm_vm_compare)) { + panic("failed to insert compatibility VM 0\n"); + } + sm_vm_compat0.state = SM_VM_STATE_AVAILABLE; +} + +status_t sm_vm_notifier_register(struct sm_vm_notifier* notif) { + spin_lock_saved_state_t state; + struct sm_vm* vm; + status_t ret; + + if (!notif) { + return ERR_INVALID_ARGS; + } + if (!notif->destroy) { + return ERR_INVALID_ARGS; + } + + spin_lock_irqsave(&sm_vm_lock, state); + sm_vm_add_compat0_locked(); + + vm = bst_search_key_type(&sm_vm_tree, ¬if->client_id, sm_vm_compare_key, + struct sm_vm, node); + if (!vm) { + ret = ERR_NOT_FOUND; + } else if (vm->state == SM_VM_STATE_AVAILABLE) { + list_add_tail(&vm->notifiers, ¬if->node); + ret = NO_ERROR; + } else { + ret = ERR_BAD_STATE; + } + spin_unlock_irqrestore(&sm_vm_lock, state); + + return ret; +} + +status_t sm_vm_notifier_unregister(struct sm_vm_notifier* notif) { + spin_lock_saved_state_t state; + status_t ret = NO_ERROR; + struct sm_vm* vm; + + if (!notif) { + return ERR_INVALID_ARGS; + } + + spin_lock_irqsave(&sm_vm_lock, state); + /* + * Check the node with the lock held to avoid + * it getting removed during the check + */ + if (!list_in_list(¬if->node)) { + ret = ERR_NOT_FOUND; + goto err_notif_not_in_list; + } + if ((uintptr_t)notif == atomic_load(&sm_vm_active_notifier)) { + spin_unlock_irqrestore(&sm_vm_lock, state); + + /* The callback is currently running, wait for it to finish */ + do { + /* + * If sm_vm_active_notifier is notif, that means that our + * notifier is currently running; retry the event_wait + * until the notifier actually completes in order to avoid + * leftover wakeups. We use a global variable because the + * notifier might have been destroyed by the handler by + * the time it returns. + */ + event_wait(&sm_vm_notifier_done_event); + } while ((uintptr_t)notif == atomic_load(&sm_vm_active_notifier)); + + /* Nothing else to do here, the notifier is already out of the list */ + return NO_ERROR; + } + + vm = bst_search_key_type(&sm_vm_tree, ¬if->client_id, sm_vm_compare_key, + struct sm_vm, node); + if (!vm) { + ret = ERR_NOT_FOUND; + goto err_no_vm; + } + + list_delete(¬if->node); + +err_notif_not_in_list: +err_no_vm: + spin_unlock_irqrestore(&sm_vm_lock, state); + return ret; +} + +static long sm_ffa_handle_framework_msg(struct smc_ret18* regs) { + uint32_t msg = regs->r2 & FFA_FRAMEWORK_MSG_MASK; + ext_mem_obj_id_t client_id = regs->r5 & 0xffffU; + struct sm_vm* vm; + long ret; + bool inserted; + + /* TODO: validate receiver */ + + switch (msg) { + case FFA_FRAMEWORK_MSG_VM_CREATED_REQ: + LTRACEF_LEVEL(1, "Got VM creation message for %" PRIu64 "\n", + client_id); + + spin_lock(&sm_vm_lock); + vm = bst_search_key_type(&sm_vm_tree, &client_id, sm_vm_compare_key, + struct sm_vm, node); + if (!vm) { + if (sm_vm_to_create == -1) { + sm_vm_to_create = client_id; + event_signal(&sm_vm_event, false); + sm_intc_raise_doorbell_irq(); + } + ret = FFA_ERROR_RETRY; + } else if (vm->state == SM_VM_STATE_FRESH) { + vm->state = SM_VM_STATE_AVAILABLE; + ret = 0; + } else { + dprintf(CRITICAL, "Duplicate VM creation for %" PRIu64 "\n", + client_id); + ret = FFA_ERROR_INVALID_PARAMETERS; + } + spin_unlock(&sm_vm_lock); + + LTRACEF_LEVEL(2, "VM creation returning %ld\n", ret); + regs->r2 = FFA_FRAMEWORK_MSG_VM_CREATED_RESP | FFA_FRAMEWORK_MSG_FLAG; + return ret; + + case FFA_FRAMEWORK_MSG_VM_DESTROYED_REQ: + LTRACEF_LEVEL(1, "Got VM destruction message for %" PRIu64 "\n", + client_id); + + spin_lock(&sm_vm_lock); + vm = bst_search_key_type(&sm_vm_tree, &client_id, sm_vm_compare_key, + struct sm_vm, node); + if (!vm) { + ret = FFA_ERROR_INVALID_PARAMETERS; + } else { + DEBUG_ASSERT(vm->state != SM_VM_STATE_READY_TO_FREE); + + switch (vm->state) { + case SM_VM_STATE_FRESH: + /* + * We got a creation request for this VM that we + * returned RETRY on, but the hypervisor never retried + * the request until we could report a success and now + * it's sending us a destruction request for that VM. + * + * We could start destroying the VM instead, but this + * is not correct hypervisor behavior so we are probably + * better off returning an error. + */ + dprintf(CRITICAL, "Got early VM destroy for %" PRIu64 "\n", + client_id); + ret = FFA_ERROR_INVALID_PARAMETERS; + break; + + case SM_VM_STATE_AVAILABLE: + vm->state = SM_VM_STATE_DESTROY_NOTIFYING; + /* + * Signal the thread so it destroys the VM and ring + * the doorbell on the host so it queues a Trusty NOP + */ + event_signal(&sm_vm_event, false); + sm_intc_raise_doorbell_irq(); + __FALLTHROUGH; + + case SM_VM_STATE_DESTROY_NOTIFYING: + ret = FFA_ERROR_RETRY; + break; + + case SM_VM_STATE_DESTROY_NOTIFIED: + /* Mark the VM for freeing since we're done with it */ + vm->state = SM_VM_STATE_READY_TO_FREE; + bst_delete(&sm_vm_tree, &vm->node); + inserted = + bst_insert(&sm_vm_free_tree, &vm->node, sm_vm_compare); + DEBUG_ASSERT(inserted); + ret = 0; + /* + * Signal the event so the VM is freed later; we do not + * need to ring the doorbell because this is not urgent, + * so the freeing can happen whenever Trusty gets cycles next. + */ + event_signal(&sm_vm_event, false); + break; + + default: + panic("Invalid VM state: %d\n", vm->state); + } + } + spin_unlock(&sm_vm_lock); + + LTRACEF_LEVEL(2, "VM destruction returning %ld\n", ret); + regs->r2 = FFA_FRAMEWORK_MSG_VM_DESTROYED_RESP | FFA_FRAMEWORK_MSG_FLAG; + return ret; + + default: + dprintf(CRITICAL, "Unhandled FF-A framework message: %x\n", msg); + return FFA_ERROR_NOT_SUPPORTED; + } +} + +static int __NO_RETURN sm_vm_notifier_loop(void* arg) { + spin_lock_saved_state_t state; + struct sm_vm* vm; + struct sm_vm_notifier* notif; + status_t ret; + + while (true) { + event_wait(&sm_vm_event); + + /* Create the new VM if a message came in */ + while (true) { + int32_t vm_id; + struct sm_vm* vm; + bool inserted; + + spin_lock_irqsave(&sm_vm_lock, state); + if (sm_vm_to_create != -1 && + sm_vm_compat0.state == SM_VM_STATE_AVAILABLE) { + /* We got an actual VM, tear down the compatibility one */ + sm_vm_compat0.state = SM_VM_STATE_DESTROY_NOTIFYING; + /* + * Signal the event so we continue the outer loop because + * the remainder of the current iteration will handle the + * new NOTIFYING state for the compatibility VM. The event + * will be used at the start of the next iteration to get + * back here and create the new VM. + */ + event_signal(&sm_vm_event, false); + /* Defer creation of the new VM until compat0 is gone */ + vm_id = -1; + } else { + vm_id = sm_vm_to_create; + } + spin_unlock_irqrestore(&sm_vm_lock, state); + + LTRACEF_LEVEL(2, "Creating fresh VM %d\n", vm_id); + if (vm_id == -1) { + break; + } + + vm = calloc(1, sizeof(struct sm_vm)); + if (!vm) { + dprintf(CRITICAL, "Out of memory for VMs\n"); + continue; + } + + vm->state = SM_VM_STATE_FRESH; + vm->client_id = vm_id; + list_initialize(&vm->notifiers); + + spin_lock_irqsave(&sm_vm_lock, state); + sm_vm_to_create = -1; + inserted = bst_insert(&sm_vm_tree, &vm->node, sm_vm_compare); + spin_unlock_irqrestore(&sm_vm_lock, state); + DEBUG_ASSERT(inserted); + } + + /* Destroy all VMs on the free list */ + while (true) { + spin_lock_irqsave(&sm_vm_lock, state); + vm = bst_next_type(&sm_vm_free_tree, NULL, struct sm_vm, node); + if (vm) { + bst_delete(&sm_vm_free_tree, &vm->node); + } + spin_unlock_irqrestore(&sm_vm_lock, state); + + if (!vm) { + break; + } + + LTRACEF_LEVEL(2, "Freeing VM %" PRIu64 "\n", vm->client_id); + DEBUG_ASSERT(vm->state == SM_VM_STATE_READY_TO_FREE); + free(vm); + } + + /* Call the next notifier */ + while (true) { + spin_lock_irqsave(&sm_vm_lock, state); + notif = NULL; + bst_for_every_entry(&sm_vm_tree, vm, struct sm_vm, node) { + if (vm->state == SM_VM_STATE_DESTROY_NOTIFYING) { + if (!list_is_empty(&vm->notifiers)) { + notif = list_remove_head_type( + &vm->notifiers, struct sm_vm_notifier, node); + atomic_store(&sm_vm_active_notifier, (uintptr_t)notif); + break; + } + + /* + * No more notifiers, we can mark the VM + * as "destroy-notified" and move on to the next one. + * + * This is thread-safe because only the current thread + * runs the notifiers, and no new nodes can be added + * while in the SM_VM_STATE_DESTROY_NOTIFYING. + */ + vm->state = SM_VM_STATE_DESTROY_NOTIFIED; + + if (vm == &sm_vm_compat0) { + /* + * We are done with compatibility VM 0, + * remove it from the tree permanently. + */ + vm->state = SM_VM_STATE_READY_TO_FREE; + bst_delete(&sm_vm_tree, &vm->node); + DEBUG_ASSERT(sm_vm_event.signaled); + } + } + } + spin_unlock_irqrestore(&sm_vm_lock, state); + + if (!notif) { + break; + } + + LTRACEF_LEVEL(2, "Calling VM destroy handler for %" PRIu64 "\n", + notif->client_id); + DEBUG_ASSERT(notif->destroy); + ret = notif->destroy(notif); + if (ret) { + TRACEF("VM destroy handler returned error (%d)\n", ret); + } + atomic_store(&sm_vm_active_notifier, 0); + event_signal(&sm_vm_notifier_done_event, true); + } + } +} + static void sm_ffa_loop(long ret, struct smc32_args* args) { struct smc_ret18 regs = {0}; uint64_t extended_args[ARM_FFA_MSG_EXTENDED_ARGS_COUNT]; @@ -354,16 +767,12 @@ } atomic_store(&sm_ffa_valid_call, true); - if (regs.r2 & (1U << 31)) { - /* TODO: support framework messages */ - dprintf(CRITICAL, "Unhandled FF-A framework message: %lx\n", - regs.r2 & 0xFFU); - regs = arm_ffa_call_error(FFA_ERROR_NOT_SUPPORTED); - break; + if (regs.r2 & FFA_FRAMEWORK_MSG_FLAG) { + ret = sm_ffa_handle_framework_msg(®s); + } else { + ret = sm_ffa_handle_direct_req(ret, ®s); } - ret = sm_ffa_handle_direct_req(ret, ®s); - LTRACEF_LEVEL(5, "Calling FFA_MSG_SEND_DIRECT_RESP (%ld)\n", ret); regs = arm_ffa_msg_send_direct_resp(®s, (ulong)ret, 0, 0, 0, 0); break; @@ -724,6 +1133,16 @@ } thread_set_real_time(stdcallthread); thread_resume(stdcallthread); + +#if LIB_SM_WITH_FFA_LOOP + sm_vm_notifier_thread = + thread_create("sm-vm-notifier", sm_vm_notifier_loop, NULL, + HIGH_PRIORITY, DEFAULT_STACK_SIZE); + if (!sm_vm_notifier_thread) { + panic("failed to create sm-vm-notifier thread!\n"); + } + thread_resume(sm_vm_notifier_thread); +#endif } LK_INIT_HOOK(libsm, sm_init, LK_INIT_LEVEL_PLATFORM - 1);