include/linux/iommufd.h - kernel/common.git - Git at Google

 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
  * Copyright (C) 2021 Intel Corporation
  * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
  */
 #ifndef __LINUX_IOMMUFD_H
 #define __LINUX_IOMMUFD_H

 #include <linux/err.h>
 #include <linux/errno.h>
 #include <linux/iommu.h>
 #include <linux/refcount.h>
 #include <linux/types.h>
 #include <linux/xarray.h>
 #include <uapi/linux/iommufd.h>

 struct device;
 struct file;
 struct iommu_group;
 struct iommu_user_data;
 struct iommu_user_data_array;
 struct iommufd_access;
 struct iommufd_ctx;
 struct iommufd_device;
 struct iommufd_viommu_ops;
 struct page;

 enum iommufd_object_type {
 	IOMMUFD_OBJ_NONE,
 	IOMMUFD_OBJ_ANY = IOMMUFD_OBJ_NONE,
 	IOMMUFD_OBJ_DEVICE,
 	IOMMUFD_OBJ_HWPT_PAGING,
 	IOMMUFD_OBJ_HWPT_NESTED,
 	IOMMUFD_OBJ_IOAS,
 	IOMMUFD_OBJ_ACCESS,
 	IOMMUFD_OBJ_FAULT,
 	IOMMUFD_OBJ_VIOMMU,
 	IOMMUFD_OBJ_VDEVICE,
 	IOMMUFD_OBJ_VEVENTQ,
 	IOMMUFD_OBJ_HW_QUEUE,
 #ifdef CONFIG_IOMMUFD_TEST
 	IOMMUFD_OBJ_SELFTEST,
 #endif
 	IOMMUFD_OBJ_MAX,
 };

 /* Base struct for all objects with a userspace ID handle. */
 struct iommufd_object {
 	/*
 	 * Destroy will sleep and wait for wait_cnt to go to zero. This allows
 	 * concurrent users of the ID to reliably avoid causing a spurious
 	 * destroy failure. Incrementing this count should either be short
 	 * lived or be revoked and blocked during pre_destroy().
 	 */
 	refcount_t wait_cnt;
 	refcount_t users;
 	enum iommufd_object_type type;
 	unsigned int id;
 };

 struct iommufd_device *iommufd_device_bind(struct iommufd_ctx *ictx,
 					   struct device *dev, u32 *id);
 void iommufd_device_unbind(struct iommufd_device *idev);

 int iommufd_device_attach(struct iommufd_device *idev, ioasid_t pasid,
 			  u32 *pt_id);
 int iommufd_device_replace(struct iommufd_device *idev, ioasid_t pasid,
 			   u32 *pt_id);
 void iommufd_device_detach(struct iommufd_device *idev, ioasid_t pasid);

 struct iommufd_ctx *iommufd_device_to_ictx(struct iommufd_device *idev);
 u32 iommufd_device_to_id(struct iommufd_device *idev);

 struct iommufd_access_ops {
 	u8 needs_pin_pages : 1;
 	void (*unmap)(void *data, unsigned long iova, unsigned long length);
 };

 enum {
 	IOMMUFD_ACCESS_RW_READ = 0,
 	IOMMUFD_ACCESS_RW_WRITE = 1 << 0,
 	/* Set if the caller is in a kthread then rw will use kthread_use_mm() */
 	IOMMUFD_ACCESS_RW_KTHREAD = 1 << 1,

 	/* Only for use by selftest */
 	__IOMMUFD_ACCESS_RW_SLOW_PATH = 1 << 2,
 };

 struct iommufd_access *
 iommufd_access_create(struct iommufd_ctx *ictx,
 		      const struct iommufd_access_ops *ops, void *data, u32 *id);
 void iommufd_access_destroy(struct iommufd_access *access);
 int iommufd_access_attach(struct iommufd_access *access, u32 ioas_id);
 int iommufd_access_replace(struct iommufd_access *access, u32 ioas_id);
 void iommufd_access_detach(struct iommufd_access *access);

 void iommufd_ctx_get(struct iommufd_ctx *ictx);

 struct iommufd_viommu {
 	struct iommufd_object obj;
 	struct iommufd_ctx *ictx;
 	struct iommu_device *iommu_dev;
 	struct iommufd_hwpt_paging *hwpt;

 	const struct iommufd_viommu_ops *ops;

 	struct xarray vdevs;
 	struct list_head veventqs;
 	struct rw_semaphore veventqs_rwsem;

 	enum iommu_viommu_type type;
 };

 struct iommufd_vdevice {
 	struct iommufd_object obj;
 	struct iommufd_viommu *viommu;
 	struct iommufd_device *idev;

 	/*
 	 * Virtual device ID per vIOMMU, e.g. vSID of ARM SMMUv3, vDeviceID of
 	 * AMD IOMMU, and vRID of Intel VT-d
 	 */
 	u64 virt_id;

 	/* Clean up all driver-specific parts of an iommufd_vdevice */
 	void (*destroy)(struct iommufd_vdevice *vdev);
 };

 struct iommufd_hw_queue {
 	struct iommufd_object obj;
 	struct iommufd_viommu *viommu;
 	struct iommufd_access *access;

 	u64 base_addr; /* in guest physical address space */
 	size_t length;

 	enum iommu_hw_queue_type type;

 	/* Clean up all driver-specific parts of an iommufd_hw_queue */
 	void (*destroy)(struct iommufd_hw_queue *hw_queue);
 };

 /**
  * struct iommufd_viommu_ops - vIOMMU specific operations
  * @destroy: Clean up all driver-specific parts of an iommufd_viommu. The memory
  *           of the vIOMMU will be free-ed by iommufd core after calling this op
  * @alloc_domain_nested: Allocate a IOMMU_DOMAIN_NESTED on a vIOMMU that holds a
  *                       nesting parent domain (IOMMU_DOMAIN_PAGING). @user_data
  *                       must be defined in include/uapi/linux/iommufd.h.
  *                       It must fully initialize the new iommu_domain before
  *                       returning. Upon failure, ERR_PTR must be returned.
  * @cache_invalidate: Flush hardware cache used by a vIOMMU. It can be used for
  *                    any IOMMU hardware specific cache: TLB and device cache.
  *                    The @array passes in the cache invalidation requests, in
  *                    form of a driver data structure. A driver must update the
  *                    array->entry_num to report the number of handled requests.
  *                    The data structure of the array entry must be defined in
  *                    include/uapi/linux/iommufd.h
  * @vdevice_size: Size of the driver-defined vDEVICE structure per this vIOMMU
  * @vdevice_init: Initialize the driver-level structure of a vDEVICE object, or
  *                related HW procedure. @vdev is already initialized by iommufd
  *                core: vdev->dev and vdev->viommu pointers; vdev->id carries a
  *                per-vIOMMU virtual ID (refer to struct iommu_vdevice_alloc in
  *                include/uapi/linux/iommufd.h)
  *                If driver has a deinit function to revert what vdevice_init op
  *                does, it should set it to the @vdev->destroy function pointer
  * @get_hw_queue_size: Get the size of a driver-defined HW queue structure for a
  *                     given @viommu corresponding to @queue_type. Driver should
  *                     return 0 if HW queue aren't supported accordingly. It is
  *                     required for driver to use the HW_QUEUE_STRUCT_SIZE macro
  *                     to sanitize the driver-level HW queue structure related
  *                     to the core one
  * @hw_queue_init_phys: Initialize the driver-level structure of a HW queue that
  *                      is initialized with its core-level structure that holds
  *                      all the info about a guest queue memory.
  *                      Driver providing this op indicates that HW accesses the
  *                      guest queue memory via physical addresses.
  *                      @index carries the logical HW QUEUE ID per vIOMMU in a
  *                      guest VM, for a multi-queue model. @base_addr_pa carries
  *                      the physical location of the guest queue
  *                      If driver has a deinit function to revert what this op
  *                      does, it should set it to the @hw_queue->destroy pointer
  */
 struct iommufd_viommu_ops {
 	void (*destroy)(struct iommufd_viommu *viommu);
 	struct iommu_domain *(*alloc_domain_nested)(
 		struct iommufd_viommu *viommu, u32 flags,
 		const struct iommu_user_data *user_data);
 	int (*cache_invalidate)(struct iommufd_viommu *viommu,
 				struct iommu_user_data_array *array);
 	const size_t vdevice_size;
 	int (*vdevice_init)(struct iommufd_vdevice *vdev);
 	size_t (*get_hw_queue_size)(struct iommufd_viommu *viommu,
 				    enum iommu_hw_queue_type queue_type);
 	/* AMD's HW will add hw_queue_init simply using @hw_queue->base_addr */
 	int (*hw_queue_init_phys)(struct iommufd_hw_queue *hw_queue, u32 index,
 				  phys_addr_t base_addr_pa);
 };

 #if IS_ENABLED(CONFIG_IOMMUFD)
 struct iommufd_ctx *iommufd_ctx_from_file(struct file *file);
 struct iommufd_ctx *iommufd_ctx_from_fd(int fd);
 void iommufd_ctx_put(struct iommufd_ctx *ictx);
 bool iommufd_ctx_has_group(struct iommufd_ctx *ictx, struct iommu_group *group);

 int iommufd_access_pin_pages(struct iommufd_access *access, unsigned long iova,
 			     unsigned long length, struct page **out_pages,
 			     unsigned int flags);
 void iommufd_access_unpin_pages(struct iommufd_access *access,
 				unsigned long iova, unsigned long length);
 int iommufd_access_rw(struct iommufd_access *access, unsigned long iova,
 		      void *data, size_t len, unsigned int flags);
 int iommufd_vfio_compat_ioas_get_id(struct iommufd_ctx *ictx, u32 *out_ioas_id);
 int iommufd_vfio_compat_ioas_create(struct iommufd_ctx *ictx);
 int iommufd_vfio_compat_set_no_iommu(struct iommufd_ctx *ictx);
 #else /* !CONFIG_IOMMUFD */
 static inline struct iommufd_ctx *iommufd_ctx_from_file(struct file *file)
 {
 	return ERR_PTR(-EOPNOTSUPP);
 }

 static inline void iommufd_ctx_put(struct iommufd_ctx *ictx)
 {
 }

 static inline int iommufd_access_pin_pages(struct iommufd_access *access,
 					   unsigned long iova,
 					   unsigned long length,
 					   struct page **out_pages,
 					   unsigned int flags)
 {
 	return -EOPNOTSUPP;
 }

 static inline void iommufd_access_unpin_pages(struct iommufd_access *access,
 					      unsigned long iova,
 					      unsigned long length)
 {
 }

 static inline int iommufd_access_rw(struct iommufd_access *access,
 				    unsigned long iova, void *data, size_t len,
 				    unsigned int flags)
 {
 	return -EOPNOTSUPP;
 }

 static inline int iommufd_vfio_compat_ioas_create(struct iommufd_ctx *ictx)
 {
 	return -EOPNOTSUPP;
 }

 static inline int iommufd_vfio_compat_set_no_iommu(struct iommufd_ctx *ictx)
 {
 	return -EOPNOTSUPP;
 }
 #endif /* CONFIG_IOMMUFD */

 #if IS_ENABLED(CONFIG_IOMMUFD_DRIVER_CORE)
 int _iommufd_object_depend(struct iommufd_object *obj_dependent,
 			   struct iommufd_object *obj_depended);
 void _iommufd_object_undepend(struct iommufd_object *obj_dependent,
 			      struct iommufd_object *obj_depended);
 int _iommufd_alloc_mmap(struct iommufd_ctx *ictx, struct iommufd_object *owner,
 			phys_addr_t mmio_addr, size_t length,
 			unsigned long *offset);
 void _iommufd_destroy_mmap(struct iommufd_ctx *ictx,
 			   struct iommufd_object *owner, unsigned long offset);
 struct device *iommufd_vdevice_to_device(struct iommufd_vdevice *vdev);
 struct device *iommufd_viommu_find_dev(struct iommufd_viommu *viommu,
 				       unsigned long vdev_id);
 int iommufd_viommu_get_vdev_id(struct iommufd_viommu *viommu,
 			       struct device *dev, unsigned long *vdev_id);
 int iommufd_viommu_report_event(struct iommufd_viommu *viommu,
 				enum iommu_veventq_type type, void *event_data,
 				size_t data_len);
 #else /* !CONFIG_IOMMUFD_DRIVER_CORE */
 static inline int _iommufd_object_depend(struct iommufd_object *obj_dependent,
 					 struct iommufd_object *obj_depended)
 {
 	return -EOPNOTSUPP;
 }

 static inline void
 _iommufd_object_undepend(struct iommufd_object *obj_dependent,
 			 struct iommufd_object *obj_depended)
 {
 }

 static inline int _iommufd_alloc_mmap(struct iommufd_ctx *ictx,
 				      struct iommufd_object *owner,
 				      phys_addr_t mmio_addr, size_t length,
 				      unsigned long *offset)
 {
 	return -EOPNOTSUPP;
 }

 static inline void _iommufd_destroy_mmap(struct iommufd_ctx *ictx,
 					 struct iommufd_object *owner,
 					 unsigned long offset)
 {
 }

 static inline struct device *
 iommufd_vdevice_to_device(struct iommufd_vdevice *vdev)
 {
 	return NULL;
 }

 static inline struct device *
 iommufd_viommu_find_dev(struct iommufd_viommu *viommu, unsigned long vdev_id)
 {
 	return NULL;
 }

 static inline int iommufd_viommu_get_vdev_id(struct iommufd_viommu *viommu,
 					     struct device *dev,
 					     unsigned long *vdev_id)
 {
 	return -ENOENT;
 }

 static inline int iommufd_viommu_report_event(struct iommufd_viommu *viommu,
 					      enum iommu_veventq_type type,
 					      void *event_data, size_t data_len)
 {
 	return -EOPNOTSUPP;
 }
 #endif /* CONFIG_IOMMUFD_DRIVER_CORE */

 #define VIOMMU_STRUCT_SIZE(drv_struct, member)                                 \
 	(sizeof(drv_struct) +                                                  \
 	 BUILD_BUG_ON_ZERO(offsetof(drv_struct, member)) +                     \
 	 BUILD_BUG_ON_ZERO(!__same_type(struct iommufd_viommu,                 \
 					((drv_struct *)NULL)->member)))

 #define VDEVICE_STRUCT_SIZE(drv_struct, member)                                \
 	(sizeof(drv_struct) +                                                  \
 	 BUILD_BUG_ON_ZERO(offsetof(drv_struct, member)) +                     \
 	 BUILD_BUG_ON_ZERO(!__same_type(struct iommufd_vdevice,                \
 					((drv_struct *)NULL)->member)))

 #define HW_QUEUE_STRUCT_SIZE(drv_struct, member)                               \
 	(sizeof(drv_struct) +                                                  \
 	 BUILD_BUG_ON_ZERO(offsetof(drv_struct, member)) +                     \
 	 BUILD_BUG_ON_ZERO(!__same_type(struct iommufd_hw_queue,               \
 					((drv_struct *)NULL)->member)))

 /*
  * Helpers for IOMMU driver to build/destroy a dependency between two sibling
  * structures created by one of the allocators above
  */
 #define iommufd_hw_queue_depend(dependent, depended, member)                   \
 	({                                                                     \
 		int ret = -EINVAL;                                             \
 									       \
 		static_assert(__same_type(struct iommufd_hw_queue,             \
 					  dependent->member));                 \
 		static_assert(__same_type(typeof(*dependent), *depended));     \
 		if (!WARN_ON_ONCE(dependent->member.viommu !=                  \
 				  depended->member.viommu))                    \
 			ret = _iommufd_object_depend(&dependent->member.obj,   \
 						     &depended->member.obj);   \
 		ret;                                                           \
 	})

 #define iommufd_hw_queue_undepend(dependent, depended, member)                 \
 	({                                                                     \
 		static_assert(__same_type(struct iommufd_hw_queue,             \
 					  dependent->member));                 \
 		static_assert(__same_type(typeof(*dependent), *depended));     \
 		WARN_ON_ONCE(dependent->member.viommu !=                       \
 			     depended->member.viommu);                         \
 		_iommufd_object_undepend(&dependent->member.obj,               \
 					 &depended->member.obj);               \
 	})

 /*
  * Helpers for IOMMU driver to alloc/destroy an mmapable area for a structure.
  *
  * To support an mmappable MMIO region, kernel driver must first register it to
  * iommufd core to allocate an @offset, during a driver-structure initialization
  * (e.g. viommu_init op). Then, it should report to user space this @offset and
  * the @length of the MMIO region for mmap syscall.
  */
 static inline int iommufd_viommu_alloc_mmap(struct iommufd_viommu *viommu,
 					    phys_addr_t mmio_addr,
 					    size_t length,
 					    unsigned long *offset)
 {
 	return _iommufd_alloc_mmap(viommu->ictx, &viommu->obj, mmio_addr,
 				   length, offset);
 }

 static inline void iommufd_viommu_destroy_mmap(struct iommufd_viommu *viommu,
 					       unsigned long offset)
 {
 	_iommufd_destroy_mmap(viommu->ictx, &viommu->obj, offset);
 }
 #endif
	/* SPDX-License-Identifier: GPL-2.0-only */
	/*
	* Copyright (C) 2021 Intel Corporation
	* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
	*/
	#ifndef __LINUX_IOMMUFD_H
	#define __LINUX_IOMMUFD_H

	#include <linux/err.h>
	#include <linux/errno.h>
	#include <linux/iommu.h>
	#include <linux/refcount.h>
	#include <linux/types.h>
	#include <linux/xarray.h>
	#include <uapi/linux/iommufd.h>

	struct device;
	struct file;
	struct iommu_group;
	struct iommu_user_data;
	struct iommu_user_data_array;
	struct iommufd_access;
	struct iommufd_ctx;
	struct iommufd_device;
	struct iommufd_viommu_ops;
	struct page;

	enum iommufd_object_type {
	IOMMUFD_OBJ_NONE,
	IOMMUFD_OBJ_ANY = IOMMUFD_OBJ_NONE,
	IOMMUFD_OBJ_DEVICE,
	IOMMUFD_OBJ_HWPT_PAGING,
	IOMMUFD_OBJ_HWPT_NESTED,
	IOMMUFD_OBJ_IOAS,
	IOMMUFD_OBJ_ACCESS,
	IOMMUFD_OBJ_FAULT,
	IOMMUFD_OBJ_VIOMMU,
	IOMMUFD_OBJ_VDEVICE,
	IOMMUFD_OBJ_VEVENTQ,
	IOMMUFD_OBJ_HW_QUEUE,
	#ifdef CONFIG_IOMMUFD_TEST
	IOMMUFD_OBJ_SELFTEST,
	#endif
	IOMMUFD_OBJ_MAX,
	};

	/* Base struct for all objects with a userspace ID handle. */
	struct iommufd_object {
	/*
	* Destroy will sleep and wait for wait_cnt to go to zero. This allows
	* concurrent users of the ID to reliably avoid causing a spurious
	* destroy failure. Incrementing this count should either be short
	* lived or be revoked and blocked during pre_destroy().
	*/
	refcount_t wait_cnt;
	refcount_t users;
	enum iommufd_object_type type;
	unsigned int id;
	};

	struct iommufd_device iommufd_device_bind(struct iommufd_ctx ictx,
	struct device dev, u32 id);
	void iommufd_device_unbind(struct iommufd_device *idev);

	int iommufd_device_attach(struct iommufd_device *idev, ioasid_t pasid,
	u32 *pt_id);
	int iommufd_device_replace(struct iommufd_device *idev, ioasid_t pasid,
	u32 *pt_id);
	void iommufd_device_detach(struct iommufd_device *idev, ioasid_t pasid);

	struct iommufd_ctx iommufd_device_to_ictx(struct iommufd_device idev);
	u32 iommufd_device_to_id(struct iommufd_device *idev);

	struct iommufd_access_ops {
	u8 needs_pin_pages : 1;
	void (unmap)(void data, unsigned long iova, unsigned long length);
	};

	enum {
	IOMMUFD_ACCESS_RW_READ = 0,
	IOMMUFD_ACCESS_RW_WRITE = 1 << 0,
	/* Set if the caller is in a kthread then rw will use kthread_use_mm() */
	IOMMUFD_ACCESS_RW_KTHREAD = 1 << 1,

	/* Only for use by selftest */
	__IOMMUFD_ACCESS_RW_SLOW_PATH = 1 << 2,
	};

	struct iommufd_access *
	iommufd_access_create(struct iommufd_ctx *ictx,
	const struct iommufd_access_ops ops, void data, u32 *id);
	void iommufd_access_destroy(struct iommufd_access *access);
	int iommufd_access_attach(struct iommufd_access *access, u32 ioas_id);
	int iommufd_access_replace(struct iommufd_access *access, u32 ioas_id);
	void iommufd_access_detach(struct iommufd_access *access);

	void iommufd_ctx_get(struct iommufd_ctx *ictx);

	struct iommufd_viommu {
	struct iommufd_object obj;
	struct iommufd_ctx *ictx;
	struct iommu_device *iommu_dev;
	struct iommufd_hwpt_paging *hwpt;

	const struct iommufd_viommu_ops *ops;

	struct xarray vdevs;
	struct list_head veventqs;
	struct rw_semaphore veventqs_rwsem;

	enum iommu_viommu_type type;
	};

	struct iommufd_vdevice {
	struct iommufd_object obj;
	struct iommufd_viommu *viommu;
	struct iommufd_device *idev;

	/*
	* Virtual device ID per vIOMMU, e.g. vSID of ARM SMMUv3, vDeviceID of
	* AMD IOMMU, and vRID of Intel VT-d
	*/
	u64 virt_id;

	/* Clean up all driver-specific parts of an iommufd_vdevice */
	void (destroy)(struct iommufd_vdevice vdev);
	};

	struct iommufd_hw_queue {
	struct iommufd_object obj;
	struct iommufd_viommu *viommu;
	struct iommufd_access *access;

	u64 base_addr; /* in guest physical address space */
	size_t length;

	enum iommu_hw_queue_type type;

	/* Clean up all driver-specific parts of an iommufd_hw_queue */
	void (destroy)(struct iommufd_hw_queue hw_queue);
	};

	/**
	* struct iommufd_viommu_ops - vIOMMU specific operations
	* @destroy: Clean up all driver-specific parts of an iommufd_viommu. The memory
	* of the vIOMMU will be free-ed by iommufd core after calling this op
	* @alloc_domain_nested: Allocate a IOMMU_DOMAIN_NESTED on a vIOMMU that holds a
	* nesting parent domain (IOMMU_DOMAIN_PAGING). @user_data
	* must be defined in include/uapi/linux/iommufd.h.
	* It must fully initialize the new iommu_domain before
	* returning. Upon failure, ERR_PTR must be returned.
	* @cache_invalidate: Flush hardware cache used by a vIOMMU. It can be used for
	* any IOMMU hardware specific cache: TLB and device cache.
	* The @array passes in the cache invalidation requests, in
	* form of a driver data structure. A driver must update the
	* array->entry_num to report the number of handled requests.
	* The data structure of the array entry must be defined in
	* include/uapi/linux/iommufd.h
	* @vdevice_size: Size of the driver-defined vDEVICE structure per this vIOMMU
	* @vdevice_init: Initialize the driver-level structure of a vDEVICE object, or
	* related HW procedure. @vdev is already initialized by iommufd
	* core: vdev->dev and vdev->viommu pointers; vdev->id carries a
	* per-vIOMMU virtual ID (refer to struct iommu_vdevice_alloc in
	* include/uapi/linux/iommufd.h)
	* If driver has a deinit function to revert what vdevice_init op
	* does, it should set it to the @vdev->destroy function pointer
	* @get_hw_queue_size: Get the size of a driver-defined HW queue structure for a
	* given @viommu corresponding to @queue_type. Driver should
	* return 0 if HW queue aren't supported accordingly. It is
	* required for driver to use the HW_QUEUE_STRUCT_SIZE macro
	* to sanitize the driver-level HW queue structure related
	* to the core one
	* @hw_queue_init_phys: Initialize the driver-level structure of a HW queue that
	* is initialized with its core-level structure that holds
	* all the info about a guest queue memory.
	* Driver providing this op indicates that HW accesses the
	* guest queue memory via physical addresses.
	* @index carries the logical HW QUEUE ID per vIOMMU in a
	* guest VM, for a multi-queue model. @base_addr_pa carries
	* the physical location of the guest queue
	* If driver has a deinit function to revert what this op
	* does, it should set it to the @hw_queue->destroy pointer
	*/
	struct iommufd_viommu_ops {
	void (destroy)(struct iommufd_viommu viommu);
	struct iommu_domain (alloc_domain_nested)(
	struct iommufd_viommu *viommu, u32 flags,
	const struct iommu_user_data *user_data);
	int (cache_invalidate)(struct iommufd_viommu viommu,
	struct iommu_user_data_array *array);
	const size_t vdevice_size;
	int (vdevice_init)(struct iommufd_vdevice vdev);
	size_t (get_hw_queue_size)(struct iommufd_viommu viommu,
	enum iommu_hw_queue_type queue_type);
	/* AMD's HW will add hw_queue_init simply using @hw_queue->base_addr */
	int (hw_queue_init_phys)(struct iommufd_hw_queue hw_queue, u32 index,
	phys_addr_t base_addr_pa);
	};

	#if IS_ENABLED(CONFIG_IOMMUFD)
	struct iommufd_ctx iommufd_ctx_from_file(struct file file);
	struct iommufd_ctx *iommufd_ctx_from_fd(int fd);
	void iommufd_ctx_put(struct iommufd_ctx *ictx);
	bool iommufd_ctx_has_group(struct iommufd_ctx ictx, struct iommu_group group);

	int iommufd_access_pin_pages(struct iommufd_access *access, unsigned long iova,
	unsigned long length, struct page **out_pages,
	unsigned int flags);
	void iommufd_access_unpin_pages(struct iommufd_access *access,
	unsigned long iova, unsigned long length);
	int iommufd_access_rw(struct iommufd_access *access, unsigned long iova,
	void *data, size_t len, unsigned int flags);
	int iommufd_vfio_compat_ioas_get_id(struct iommufd_ctx ictx, u32 out_ioas_id);
	int iommufd_vfio_compat_ioas_create(struct iommufd_ctx *ictx);
	int iommufd_vfio_compat_set_no_iommu(struct iommufd_ctx *ictx);
	#else /* !CONFIG_IOMMUFD */
	static inline struct iommufd_ctx iommufd_ctx_from_file(struct file file)
	{
	return ERR_PTR(-EOPNOTSUPP);
	}

	static inline void iommufd_ctx_put(struct iommufd_ctx *ictx)
	{
	}

	static inline int iommufd_access_pin_pages(struct iommufd_access *access,
	unsigned long iova,
	unsigned long length,
	struct page **out_pages,
	unsigned int flags)
	{
	return -EOPNOTSUPP;
	}

	static inline void iommufd_access_unpin_pages(struct iommufd_access *access,
	unsigned long iova,
	unsigned long length)
	{
	}

	static inline int iommufd_access_rw(struct iommufd_access *access,
	unsigned long iova, void *data, size_t len,
	unsigned int flags)
	{
	return -EOPNOTSUPP;
	}

	static inline int iommufd_vfio_compat_ioas_create(struct iommufd_ctx *ictx)
	{
	return -EOPNOTSUPP;
	}

	static inline int iommufd_vfio_compat_set_no_iommu(struct iommufd_ctx *ictx)
	{
	return -EOPNOTSUPP;
	}
	#endif /* CONFIG_IOMMUFD */

	#if IS_ENABLED(CONFIG_IOMMUFD_DRIVER_CORE)
	int _iommufd_object_depend(struct iommufd_object *obj_dependent,
	struct iommufd_object *obj_depended);
	void _iommufd_object_undepend(struct iommufd_object *obj_dependent,
	struct iommufd_object *obj_depended);
	int _iommufd_alloc_mmap(struct iommufd_ctx ictx, struct iommufd_object owner,
	phys_addr_t mmio_addr, size_t length,
	unsigned long *offset);
	void _iommufd_destroy_mmap(struct iommufd_ctx *ictx,
	struct iommufd_object *owner, unsigned long offset);
	struct device iommufd_vdevice_to_device(struct iommufd_vdevice vdev);
	struct device iommufd_viommu_find_dev(struct iommufd_viommu viommu,
	unsigned long vdev_id);
	int iommufd_viommu_get_vdev_id(struct iommufd_viommu *viommu,
	struct device dev, unsigned long vdev_id);
	int iommufd_viommu_report_event(struct iommufd_viommu *viommu,
	enum iommu_veventq_type type, void *event_data,
	size_t data_len);
	#else /* !CONFIG_IOMMUFD_DRIVER_CORE */
	static inline int _iommufd_object_depend(struct iommufd_object *obj_dependent,
	struct iommufd_object *obj_depended)
	{
	return -EOPNOTSUPP;
	}

	static inline void
	_iommufd_object_undepend(struct iommufd_object *obj_dependent,
	struct iommufd_object *obj_depended)
	{
	}

	static inline int _iommufd_alloc_mmap(struct iommufd_ctx *ictx,
	struct iommufd_object *owner,
	phys_addr_t mmio_addr, size_t length,
	unsigned long *offset)
	{
	return -EOPNOTSUPP;
	}

	static inline void _iommufd_destroy_mmap(struct iommufd_ctx *ictx,
	struct iommufd_object *owner,
	unsigned long offset)
	{
	}

	static inline struct device *
	iommufd_vdevice_to_device(struct iommufd_vdevice *vdev)
	{
	return NULL;
	}

	static inline struct device *
	iommufd_viommu_find_dev(struct iommufd_viommu *viommu, unsigned long vdev_id)
	{
	return NULL;
	}

	static inline int iommufd_viommu_get_vdev_id(struct iommufd_viommu *viommu,
	struct device *dev,
	unsigned long *vdev_id)
	{
	return -ENOENT;
	}

	static inline int iommufd_viommu_report_event(struct iommufd_viommu *viommu,
	enum iommu_veventq_type type,
	void *event_data, size_t data_len)
	{
	return -EOPNOTSUPP;
	}
	#endif /* CONFIG_IOMMUFD_DRIVER_CORE */

	#define VIOMMU_STRUCT_SIZE(drv_struct, member) \
	(sizeof(drv_struct) + \
	BUILD_BUG_ON_ZERO(offsetof(drv_struct, member)) + \
	BUILD_BUG_ON_ZERO(!__same_type(struct iommufd_viommu, \
	((drv_struct *)NULL)->member)))

	#define VDEVICE_STRUCT_SIZE(drv_struct, member) \
	(sizeof(drv_struct) + \
	BUILD_BUG_ON_ZERO(offsetof(drv_struct, member)) + \
	BUILD_BUG_ON_ZERO(!__same_type(struct iommufd_vdevice, \
	((drv_struct *)NULL)->member)))

	#define HW_QUEUE_STRUCT_SIZE(drv_struct, member) \
	(sizeof(drv_struct) + \
	BUILD_BUG_ON_ZERO(offsetof(drv_struct, member)) + \
	BUILD_BUG_ON_ZERO(!__same_type(struct iommufd_hw_queue, \
	((drv_struct *)NULL)->member)))

	/*
	* Helpers for IOMMU driver to build/destroy a dependency between two sibling
	* structures created by one of the allocators above
	*/
	#define iommufd_hw_queue_depend(dependent, depended, member) \
	({ \
	int ret = -EINVAL; \
	\
	static_assert(__same_type(struct iommufd_hw_queue, \
	dependent->member)); \
	static_assert(__same_type(typeof(dependent), depended)); \
	if (!WARN_ON_ONCE(dependent->member.viommu != \
	depended->member.viommu)) \
	ret = _iommufd_object_depend(&dependent->member.obj, \
	&depended->member.obj); \
	ret; \
	})

	#define iommufd_hw_queue_undepend(dependent, depended, member) \
	({ \
	static_assert(__same_type(struct iommufd_hw_queue, \
	dependent->member)); \
	static_assert(__same_type(typeof(dependent), depended)); \
	WARN_ON_ONCE(dependent->member.viommu != \
	depended->member.viommu); \
	_iommufd_object_undepend(&dependent->member.obj, \
	&depended->member.obj); \
	})

	/*
	* Helpers for IOMMU driver to alloc/destroy an mmapable area for a structure.
	*
	* To support an mmappable MMIO region, kernel driver must first register it to
	* iommufd core to allocate an @offset, during a driver-structure initialization
	* (e.g. viommu_init op). Then, it should report to user space this @offset and
	* the @length of the MMIO region for mmap syscall.
	*/
	static inline int iommufd_viommu_alloc_mmap(struct iommufd_viommu *viommu,
	phys_addr_t mmio_addr,
	size_t length,
	unsigned long *offset)
	{
	return _iommufd_alloc_mmap(viommu->ictx, &viommu->obj, mmio_addr,
	length, offset);
	}

	static inline void iommufd_viommu_destroy_mmap(struct iommufd_viommu *viommu,
	unsigned long offset)
	{
	_iommufd_destroy_mmap(viommu->ictx, &viommu->obj, offset);
	}
	#endif