runtime/Callbacks.h

/*
 * Copyright (C) 2017 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#ifndef ANDROID_HARDWARE_NEURALNETWORKS_V1_0_CALLBACKS_H
#define ANDROID_HARDWARE_NEURALNETWORKS_V1_0_CALLBACKS_H

#include <android-base/thread_annotations.h>
#include <android/hardware/neuralnetworks/1.0/IExecutionCallback.h>
#include <android/hardware/neuralnetworks/1.0/IPreparedModelCallback.h>
#include <android/hardware/neuralnetworks/1.2/IExecutionCallback.h>
#include <android/hardware/neuralnetworks/1.2/IPreparedModelCallback.h>
#include <hidl/MQDescriptor.h>
#include <hidl/Status.h>
#include <condition_variable>
#include <functional>
#include <mutex>
#include <thread>

namespace android {
namespace hardware {
namespace neuralnetworks {
namespace V1_2 {
namespace implementation {

using V1_0::ErrorStatus;

/**
 * The CallbackBase class is used internally by the NeuralNetworks runtime to
 * synchronize between different threads. An asynchronous task is launched
 * paired with a callback object. When a client thread requires the output being
 * generated by the asynchronous task, the client thread can wait for the result
 * and be blocked until it has completed. Any wait may safely be called
 * concurrently, even on the same callback object. When the asynchronous task
 * has finished its workload, it must immediately call "notify". If the
 * asynchronous task has failed to launch, the function that tried to launch the
 * asynchronous task must immediately call "notify". This "notify" call awakens
 * any client threads waiting on the callback object.
 *
 * The CallbackBase class implements some of the base synchronization common to
 * both PrepareModelCallback and ExecutionCallback. For consistency, any HIDL
 * callback class must inherit from CallbackBase as well as the HIDL callback
 * interface it implements.
 *
 * This class exists to enable synchronization across HIDL. When synchronization
 * is only required in the same process, consider using std::future, std::mutex,
 * std::condition_variable, or std::experimental::latch instead.
 */
class CallbackBase {
   public:
    CallbackBase();
    ~CallbackBase();

    /**
     * CallbackBase::wait blocks until notify has been called on the callback
     * object.
     */
    void wait();

    /**
     * CallbackBase::bind_thread binds a thread to the callback object for later
     * use by CallbackBase::join_thread_locked.
     *
     * The thread must be passed using std::move.
     *
     * Once a thread is bound with CallbackBase::bind_thread, the client code
     * should ensure that CallbackBase::wait has been called before the callback
     * object is destroyed.
     *
     * The bound thread shall not call any CallbackBase method with the
     * exception of CallbackBase::notify, which it must call when the thread has
     * finished its computation.
     *
     * CallbackBase::bind_thread can be called at most once on a given callback
     * object.
     *
     * @param asyncThread Thread to be bound to the callback object. The thread
     *     object must represent a thread of execution -- i.e.,
     *     asyncThread.joinable() must be true.
     * @return bool True if successful, false if thread was not properly bound.
     */
    bool bind_thread(std::thread&& asyncThread);

   protected:
    /**
     * CallbackBase::notify enables all prior and future wait calls on the
     * callback object to proceed. The call to CallbackBase::notify happens
     * before any wait calls on this callback object return. The asynchronous
     * call the callback object is paired with must ensure that any update to
     * state that should be visible to the caller of wait happens before the
     * call to CallbackBase::notify.
     *
     * CallbackBase::notify must be called exactly once on a given callback
     * object.
     */
    void notify();

    /**
     * CallbackBase::on_finish binds a function to the callback object. This
     * bound function will be executed when CallbackBase::notify is called,
     * before any calls to wait return.
     *
     * The bound function must not synchronize with or otherwise access the
     * callback object it is bound to, as this could cause a deadlock.
     *
     * CallbackBase::on_finish can be called at most once on a given callback
     * object, and the call to CallbackBase::on_finish must finish before
     * CallbackBase::notify is called.
     *
     * @param post_work Function to be invoked the first time
     *     CallbackBase::notify is called. Must have a target -- i.e., must not
     *     compare equal to nullptr.
     * @return bool True if the function was successfully bound, false if
     *     unsuccessful.
     */
    bool on_finish(std::function<void()> post_work);

   private:
    /**
     * CallbackBase::join_thread_locked ensures that the thread (if any) bound
     * to this callback object with CallbackBase::bind_thread has fully finished
     * and cleaned its resources.
     *
     * CallbackBase::join_thread_locked can be called multiple times. When
     * called, it must be called while the object's mutex is locked.
     */
    void join_thread_locked();

    bool mNotified;
    std::mutex mMutex;
    std::condition_variable mCondition;
    std::function<void()> mPostWork;
    std::thread mThread;
};

/**
 * The PreparedModelCallback class is used to receive the error status of
 * preparing a model as well as the prepared model from a task executing
 * asynchronously with respect to the runtime. If a calling thread calls wait
 * or get* on a PreparedModelCallback object and the corresponding asynchronous
 * task has not finished preparing the model, the calling thread will block
 * until the asynchronous task has either called notify or notify_1_2.
 *
 * If the callback object is notified more than once, only the results of the
 * first call to notify are used, and the results from subsequent calls are
 * discarded.
 *
 * This callback object is passed as an argument to IDevice::prepareModel*.
 */
class PreparedModelCallback : public IPreparedModelCallback {
   public:
    /**
     * IPreparedModelCallback::notify marks the callback object with the return
     * status of the asynchronous model preparation along with the prepared
     * model, and allows all prior and future wait calls on the
     * PreparedModelCallback object to proceed.
     *
     * Either IPreparedModelCallback::notify or
     * IPreparedModelCallback::notify_1_2 must be called on a given
     * PreparedModelCallback object.
     *
     * If the callback object is notified more than once, only the results of
     * the first call to notify is used, and the results from subsequent calls
     * are discarded.
     *
     * @param status Error status returned from asynchronously preparing the
     *     model; will be:
     *     - NONE if the asynchronous preparation was successful
     *     - DEVICE_UNAVAILABLE if driver is offline or busy
     *     - GENERAL_FAILURE if there is an unspecified error
     *     - INVALID_ARGUMENT if the input model is invalid
     * @param preparedModel Returned model that has been prepared for execution,
     *     nullptr if the model was unable to be prepared.
     */
    Return<void> notify(ErrorStatus status, const sp<V1_0::IPreparedModel>& preparedModel) override;

    /**
     * IPreparedModelCallback::notify_1_2 marks the callback object with the
     * return status of the asynchronous model preparation along with the
     * prepared model, and allows all prior and future wait calls on the
     * PreparedModelCallback object to proceed.
     *
     * Either IPreparedModelCallback::notify or
     * IPreparedModelCallback::notify_1_2 must be called on a given
     * PreparedModelCallback object.
     *
     * If the callback object is notified more than once, only the results of
     * the first call to notify are used, and the results from subsequent calls
     * are discarded.
     *
     * @param status Error status returned from asynchronously preparing the
     *     model; will be:
     *     - NONE if the asynchronous preparation was successful
     *     - DEVICE_UNAVAILABLE if driver is offline or busy
     *     - GENERAL_FAILURE if there is an unspecified error
     *     - INVALID_ARGUMENT if the input model is invalid
     * @param preparedModel Returned model that has been prepared for execution,
     *     nullptr if the model was unable to be prepared.
     */
    Return<void> notify_1_2(ErrorStatus status,
                            const sp<V1_2::IPreparedModel>& preparedModel) override;

    /**
     * PreparedModelCallback::wait blocks until notify* has been called on the
     * callback object.
     */
    void wait() const;

    /**
     * Retrieves the error status returned from the asynchronous task launched
     * by IDevice::prepareModel*. If IDevice::prepareModel* has not finished
     * asynchronously preparing the model, this call will block until the
     * asynchronous task notifies the object.
     *
     * @return status Error status returned from asynchronously preparing the
     *     model; will be:
     *     - NONE if the asynchronous preparation was successful
     *     - DEVICE_UNAVAILABLE if driver is offline or busy
     *     - GENERAL_FAILURE if there is an unspecified error
     *     - INVALID_ARGUMENT if the input model is invalid
     */
    ErrorStatus getStatus() const;

    /**
     * Retrieves the model that has been prepared for execution from the
     * asynchronous task launched by IDevice::prepareModel*. If
     * IDevice::prepareModel* has not finished asynchronously preparing the
     * model, this call will block until the asynchronous task notifies the
     * object.
     *
     * @return preparedModel Returned model that has been prepared for
     *     execution, nullptr if the model was unable to be prepared.
     */
    sp<V1_0::IPreparedModel> getPreparedModel() const;

   private:
    mutable std::mutex mMutex;
    mutable std::condition_variable mCondition;
    bool mNotified GUARDED_BY(mMutex) = false;
    ErrorStatus mErrorStatus = ErrorStatus::GENERAL_FAILURE;
    sp<V1_0::IPreparedModel> mPreparedModel;
};

/**
 * The ExecutionCallback class is used to receive the error status of the
 * execution from a task executing asynchronously with respect to the runtime.
 * If a calling thread calls wait or get* on a PreparedModelCallback object and
 * the corresponding asynchronous task has not finished the execution, the
 * calling thread will block until the asynchronous task has either called notify
 * or notify_1_2. For more information on the synchronization behavior, refer to
 * the CallbackBase class.
 *
 * This class inherits the basic blocking and signaling calls from
 * CallbackBase, and implements the HIDL notify and notify_1_2 calls from
 * IExecutionCallback. This callback object is passed as an argument to
 * IPreparedModel::execute.
 */
class ExecutionCallback : public CallbackBase, public IExecutionCallback {
    using ExecutionFinish =
            std::function<ErrorStatus(ErrorStatus, const std::vector<OutputShape>&)>;

   public:
    ExecutionCallback();
    ~ExecutionCallback() override;

    /**
     * IExecutionCallback::notify and IExecutionCallback::notify_1_2 mark the
     * callback object with the return status of the asynchronous execution that
     * held this callback and enable all prior and future wait calls on the
     * ExecutionCallback object to proceed. For more information on the
     * synchronization behavior, refer to the CallbackBase class.
     *
     * Either IExecutionCallback::notify or IExecutionCallback::notify_1_2 must
     * be called exactly once on a given ExecutionCallback object.
     *
     * @param status Error status returned from launching the asynchronous task
     *     (if the launch fails) or from the asynchronous task itself (if the
     *     launch succeeds). Must be:
     *     - NONE if the asynchronous execution was successful
     *     - DEVICE_UNAVAILABLE if driver is offline or busy
     *     - GENERAL_FAILURE if there is an unspecified error
     *     - OUTPUT_INSUFFICIENT_SIZE if provided output buffer is not large
     *         enough to store the resultant values
     *     - INVALID_ARGUMENT if the input request is invalid
     */
    Return<void> notify(ErrorStatus status) override;

    /**
     * Similar to IExecutionCallback::notify, but for V1_2::IPreparedModel to
     * also notify output shapes along with error status.
     *
     * @param status Error status returned from launching the asynchronous task
     *     (if the launch fails) or from the asynchronous task itself (if the
     *     launch succeeds). Must be:
     *     - NONE if the asynchronous execution was successful
     *     - DEVICE_UNAVAILABLE if driver is offline or busy
     *     - GENERAL_FAILURE if the asynchronous task resulted in an unspecified
     *         error
     *     - OUTPUT_INSUFFICIENT_SIZE if at least one output operand buffer is
     *         not large enough to store the corresponding output
     *     - INVALID_ARGUMENT if one of the input arguments to prepareModel is
     *         invalid
     * @param outputShapes A list of shape information of model output operands.
     *     The index into "outputShapes" corresponds to the index of the output
     *     operand in the Request outputs vector. outputShapes must be empty
     *     unless the status is either NONE or OUTPUT_INSUFFICIENT_SIZE.
     * @return Timing Duration of execution. Unless MeasureTiming::YES was
     *     passed when launching the execution and status is NONE, all times
     *     must be reported as UINT64_MAX. A driver may choose to report any
     *     time as UINT64_MAX, indicating that particular measurement is not
     *     available.
     */
    Return<void> notify_1_2(ErrorStatus status, const hidl_vec<OutputShape>& outputShapes,
                            const Timing& timing) override;

    // An overload of the latest notify interface to hide the version from ExecutionBuilder.
    Return<void> notify(ErrorStatus status, const hidl_vec<OutputShape>& outputShapes,
                        const Timing& timing) {
        return notify_1_2(status, outputShapes, timing);
    }

    /**
     * Retrieves the error status returned from the asynchronous task launched
     * by either IPreparedModel::execute or IPreparedModel::execute_1_2. If
     * IPreparedModel::execute or IPreparedModel::execute_1_2 has not finished
     * asynchronously executing, this call will block until the asynchronous
     * task notifies the object.
     *
     * @return status Error status returned from launching the asynchronous task
     *     (if the launch fails) or from the asynchronous task itself (if the
     *     launch succeeds). Must be:
     *     - NONE if the asynchronous execution was successful
     *     - DEVICE_UNAVAILABLE if driver is offline or busy
     *     - GENERAL_FAILURE if the asynchronous task resulted in an unspecified
     *         error
     *     - OUTPUT_INSUFFICIENT_SIZE if at least one output operand buffer is
     *         not large enough to store the corresponding output
     *     - INVALID_ARGUMENT if one of the input arguments to prepareModel is
     *         invalid
     */
    ErrorStatus getStatus();

    /**
     * Retrieves the output shapes returned from the asynchronous task launched
     * by IPreparedModel::execute_1_2. If IPreparedModel::execute_1_2 has not
     * finished asynchronously executing, this call will block until the
     * asynchronous task notifies the object.
     *
     * If the asynchronous task was launched by IPreparedModel::execute, an
     * empty vector will be returned.
     *
     * @return outputShapes A list of shape information of model output
     *     operands. The index into "outputShapes" corresponds to the index of
     *     the output operand in the Request outputs vector. outputShapes must
     *     be empty unless the status is either NONE or
     *     OUTPUT_INSUFFICIENT_SIZE. outputShaps may be empty if the status is
     *     NONE and all model output operands are fully-specified at execution
     *     time. outputShapes must have the same number of elements as the
     *     number of model output operands if the status is
     *     OUTPUT_INSUFFICIENT_SIZE, or if the status is NONE and the model has
     *     at least one output operand that is not fully-specified.
     */
    const std::vector<OutputShape>& getOutputShapes();

    /**
     * Retrieves the duration of execution of the asynchronous task launched by
     * IPreparedModel::execute_1_2. If IPreparedModel::execute_1_2 has not
     * finished asynchronously executing, this call will block until the
     * asynchronous task notifies the object.
     *
     * If the asynchronous task was launched by IPreparedModel::execute, every
     * time must be UINT64_MAX.
     *
     * @return timing Duration of the execution. Every time must be UINT64_MAX
     *     unless the status is NONE.
     */
    Timing getTiming();

    // The callback will invoke finish(mErrorStatus) on finish.
    void setOnFinish(const ExecutionFinish& finish) { mOnFinish = finish; }

   private:
    ErrorStatus mErrorStatus = ErrorStatus::GENERAL_FAILURE;
    std::vector<OutputShape> mOutputShapes = {};
    Timing mTiming = {};
    ExecutionFinish mOnFinish;
};

}  // namespace implementation
}  // namespace V1_2
}  // namespace neuralnetworks
}  // namespace hardware
}  // namespace android

namespace android::nn {

using ::android::hardware::neuralnetworks::V1_2::implementation::ExecutionCallback;
using ::android::hardware::neuralnetworks::V1_2::implementation::PreparedModelCallback;

}  // namespace android::nn

#endif  // ANDROID_HARDWARE_NEURALNETWORKS_V1_0_CALLBACKS_H