Merge "Merge e3c2b10d6f15 ("Merge tag 'arm64-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux") into android-mainline" into android-mainlinediff --git a/Documentation/ABI/testing/sysfs-bus-cxl b/Documentation/ABI/testing/sysfs-bus-cxl
index 48ac0d9..6350dd8 100644
--- a/Documentation/ABI/testing/sysfs-bus-cxl
+++ b/Documentation/ABI/testing/sysfs-bus-cxl
@@ -58,6 +58,54 @@
affinity for this device.
+What: /sys/bus/cxl/devices/memX/security/state
+Date: June, 2023
+KernelVersion: v6.5
+Contact: linux-cxl@vger.kernel.org
+Description:
+ (RO) Reading this file will display the CXL security state for
+ that device. Such states can be: 'disabled', 'sanitize', when
+ a sanitization is currently underway; or those available only
+ for persistent memory: 'locked', 'unlocked' or 'frozen'. This
+ sysfs entry is select/poll capable from userspace to notify
+ upon completion of a sanitize operation.
+
+
+What: /sys/bus/cxl/devices/memX/security/sanitize
+Date: June, 2023
+KernelVersion: v6.5
+Contact: linux-cxl@vger.kernel.org
+Description:
+ (WO) Write a boolean 'true' string value to this attribute to
+ sanitize the device to securely re-purpose or decommission it.
+ This is done by ensuring that all user data and meta-data,
+ whether it resides in persistent capacity, volatile capacity,
+ or the LSA, is made permanently unavailable by whatever means
+ is appropriate for the media type. This functionality requires
+ the device to be not be actively decoding any HPA ranges.
+
+
+What /sys/bus/cxl/devices/memX/security/erase
+Date: June, 2023
+KernelVersion: v6.5
+Contact: linux-cxl@vger.kernel.org
+Description:
+ (WO) Write a boolean 'true' string value to this attribute to
+ secure erase user data by changing the media encryption keys for
+ all user data areas of the device.
+
+
+What: /sys/bus/cxl/devices/memX/firmware/
+Date: April, 2023
+KernelVersion: v6.5
+Contact: linux-cxl@vger.kernel.org
+Description:
+ (RW) Firmware uploader mechanism. The different files under
+ this directory can be used to upload and activate new
+ firmware for CXL devices. The interfaces under this are
+ documented in sysfs-class-firmware.
+
+
What: /sys/bus/cxl/devices/*/devtype
Date: June, 2021
KernelVersion: v5.14
diff --git a/Documentation/admin-guide/perf/cxl.rst b/Documentation/admin-guide/perf/cxl.rst
new file mode 100644
index 0000000..9233ea0
--- /dev/null
+++ b/Documentation/admin-guide/perf/cxl.rst
@@ -0,0 +1,68 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+======================================
+CXL Performance Monitoring Unit (CPMU)
+======================================
+
+The CXL rev 3.0 specification provides a definition of CXL Performance
+Monitoring Unit in section 13.2: Performance Monitoring.
+
+CXL components (e.g. Root Port, Switch Upstream Port, End Point) may have
+any number of CPMU instances. CPMU capabilities are fully discoverable from
+the devices. The specification provides event definitions for all CXL protocol
+message types and a set of additional events for things commonly counted on
+CXL devices (e.g. DRAM events).
+
+CPMU driver
+===========
+
+The CPMU driver registers a perf PMU with the name pmu_mem<X>.<Y> on the CXL bus
+representing the Yth CPMU for memX.
+
+ /sys/bus/cxl/device/pmu_mem<X>.<Y>
+
+The associated PMU is registered as
+
+ /sys/bus/event_sources/devices/cxl_pmu_mem<X>.<Y>
+
+In common with other CXL bus devices, the id has no specific meaning and the
+relationship to specific CXL device should be established via the device parent
+of the device on the CXL bus.
+
+PMU driver provides description of available events and filter options in sysfs.
+
+The "format" directory describes all formats of the config (event vendor id,
+group id and mask) config1 (threshold, filter enables) and config2 (filter
+parameters) fields of the perf_event_attr structure. The "events" directory
+describes all documented events show in perf list.
+
+The events shown in perf list are the most fine grained events with a single
+bit of the event mask set. More general events may be enable by setting
+multiple mask bits in config. For example, all Device to Host Read Requests
+may be captured on a single counter by setting the bits for all of
+
+* d2h_req_rdcurr
+* d2h_req_rdown
+* d2h_req_rdshared
+* d2h_req_rdany
+* d2h_req_rdownnodata
+
+Example of usage::
+
+ $#perf list
+ cxl_pmu_mem0.0/clock_ticks/ [Kernel PMU event]
+ cxl_pmu_mem0.0/d2h_req_rdshared/ [Kernel PMU event]
+ cxl_pmu_mem0.0/h2d_req_snpcur/ [Kernel PMU event]
+ cxl_pmu_mem0.0/h2d_req_snpdata/ [Kernel PMU event]
+ cxl_pmu_mem0.0/h2d_req_snpinv/ [Kernel PMU event]
+ -----------------------------------------------------------
+
+ $# perf stat -a -e cxl_pmu_mem0.0/clock_ticks/ -e cxl_pmu_mem0.0/d2h_req_rdshared/
+
+Vendor specific events may also be available and if so can be used via
+
+ $# perf stat -a -e cxl_pmu_mem0.0/vid=VID,gid=GID,mask=MASK/
+
+The driver does not support sampling so "perf record" is unsupported.
+It only supports system-wide counting so attaching to a task is
+unsupported.
diff --git a/Documentation/admin-guide/perf/index.rst b/Documentation/admin-guide/perf/index.rst
index 9de64a4..f60be04 100644
--- a/Documentation/admin-guide/perf/index.rst
+++ b/Documentation/admin-guide/perf/index.rst
@@ -21,3 +21,4 @@
alibaba_pmu
nvidia-pmu
meson-ddr-pmu
+ cxl
diff --git a/MAINTAINERS b/MAINTAINERS
index 4d094ef..316cbc4 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -5203,6 +5203,13 @@
F: drivers/cxl/
F: include/uapi/linux/cxl_mem.h
+COMPUTE EXPRESS LINK PMU (CPMU)
+M: Jonathan Cameron <jonathan.cameron@huawei.com>
+L: linux-cxl@vger.kernel.org
+S: Maintained
+F: Documentation/admin-guide/perf/cxl.rst
+F: drivers/perf/cxl_pmu.c
+
CONEXANT ACCESSRUNNER USB DRIVER
L: accessrunner-general@lists.sourceforge.net
S: Orphan
diff --git a/arch/arm64/include/asm/fpsimdmacros.h b/arch/arm64/include/asm/fpsimdmacros.h
index cd03819..cdf6a35 100644
--- a/arch/arm64/include/asm/fpsimdmacros.h
+++ b/arch/arm64/include/asm/fpsimdmacros.h
@@ -316,12 +316,12 @@
_for n, 0, 15, _sve_str_p \n, \nxbase, \n - 16
cbz \save_ffr, 921f
_sve_rdffr 0
- _sve_str_p 0, \nxbase
- _sve_ldr_p 0, \nxbase, -16
b 922f
921:
- str xzr, [x\nxbase] // Zero out FFR
+ _sve_pfalse 0 // Zero out FFR
922:
+ _sve_str_p 0, \nxbase
+ _sve_ldr_p 0, \nxbase, -16
mrs x\nxtmp, fpsr
str w\nxtmp, [\xpfpsr]
mrs x\nxtmp, fpcr
diff --git a/drivers/cxl/Kconfig b/drivers/cxl/Kconfig
index ff4e7811..fcbf829 100644
--- a/drivers/cxl/Kconfig
+++ b/drivers/cxl/Kconfig
@@ -82,6 +82,7 @@
config CXL_MEM
tristate "CXL: Memory Expansion"
depends on CXL_PCI
+ select FW_UPLOAD
default CXL_BUS
help
The CXL.mem protocol allows a device to act as a provider of "System
@@ -139,4 +140,17 @@
If unsure, or if this kernel is meant for production environments,
say N.
+config CXL_PMU
+ tristate "CXL Performance Monitoring Unit"
+ default CXL_BUS
+ depends on PERF_EVENTS
+ help
+ Support performance monitoring as defined in CXL rev 3.0
+ section 13.2: Performance Monitoring. CXL components may have
+ one or more CXL Performance Monitoring Units (CPMUs).
+
+ Say 'y/m' to enable a driver that will attach to performance
+ monitoring units and provide standard perf based interfaces.
+
+ If unsure say 'm'.
endif
diff --git a/drivers/cxl/acpi.c b/drivers/cxl/acpi.c
index 7e1765b..658e6b8 100644
--- a/drivers/cxl/acpi.c
+++ b/drivers/cxl/acpi.c
@@ -258,7 +258,7 @@ static int cxl_parse_cfmws(union acpi_subtable_headers *header, void *arg,
cxld = &cxlrd->cxlsd.cxld;
cxld->flags = cfmws_to_decoder_flags(cfmws->restrictions);
- cxld->target_type = CXL_DECODER_EXPANDER;
+ cxld->target_type = CXL_DECODER_HOSTONLYMEM;
cxld->hpa_range = (struct range) {
.start = res->start,
.end = res->end,
@@ -327,6 +327,120 @@ __mock struct acpi_device *to_cxl_host_bridge(struct device *host,
return NULL;
}
+/* Note, @dev is used by mock_acpi_table_parse_cedt() */
+struct cxl_chbs_context {
+ struct device *dev;
+ unsigned long long uid;
+ resource_size_t base;
+ u32 cxl_version;
+};
+
+static int cxl_get_chbs_iter(union acpi_subtable_headers *header, void *arg,
+ const unsigned long end)
+{
+ struct cxl_chbs_context *ctx = arg;
+ struct acpi_cedt_chbs *chbs;
+
+ if (ctx->base != CXL_RESOURCE_NONE)
+ return 0;
+
+ chbs = (struct acpi_cedt_chbs *) header;
+
+ if (ctx->uid != chbs->uid)
+ return 0;
+
+ ctx->cxl_version = chbs->cxl_version;
+ if (!chbs->base)
+ return 0;
+
+ if (chbs->cxl_version == ACPI_CEDT_CHBS_VERSION_CXL11 &&
+ chbs->length != CXL_RCRB_SIZE)
+ return 0;
+
+ ctx->base = chbs->base;
+
+ return 0;
+}
+
+static int cxl_get_chbs(struct device *dev, struct acpi_device *hb,
+ struct cxl_chbs_context *ctx)
+{
+ unsigned long long uid;
+ int rc;
+
+ rc = acpi_evaluate_integer(hb->handle, METHOD_NAME__UID, NULL, &uid);
+ if (rc != AE_OK) {
+ dev_err(dev, "unable to retrieve _UID\n");
+ return -ENOENT;
+ }
+
+ dev_dbg(dev, "UID found: %lld\n", uid);
+ *ctx = (struct cxl_chbs_context) {
+ .dev = dev,
+ .uid = uid,
+ .base = CXL_RESOURCE_NONE,
+ .cxl_version = UINT_MAX,
+ };
+
+ acpi_table_parse_cedt(ACPI_CEDT_TYPE_CHBS, cxl_get_chbs_iter, ctx);
+
+ return 0;
+}
+
+static int add_host_bridge_dport(struct device *match, void *arg)
+{
+ acpi_status rc;
+ struct device *bridge;
+ struct cxl_dport *dport;
+ struct cxl_chbs_context ctx;
+ struct acpi_pci_root *pci_root;
+ struct cxl_port *root_port = arg;
+ struct device *host = root_port->dev.parent;
+ struct acpi_device *hb = to_cxl_host_bridge(host, match);
+
+ if (!hb)
+ return 0;
+
+ rc = cxl_get_chbs(match, hb, &ctx);
+ if (rc)
+ return rc;
+
+ if (ctx.cxl_version == UINT_MAX) {
+ dev_warn(match, "No CHBS found for Host Bridge (UID %lld)\n",
+ ctx.uid);
+ return 0;
+ }
+
+ if (ctx.base == CXL_RESOURCE_NONE) {
+ dev_warn(match, "CHBS invalid for Host Bridge (UID %lld)\n",
+ ctx.uid);
+ return 0;
+ }
+
+ pci_root = acpi_pci_find_root(hb->handle);
+ bridge = pci_root->bus->bridge;
+
+ /*
+ * In RCH mode, bind the component regs base to the dport. In
+ * VH mode it will be bound to the CXL host bridge's port
+ * object later in add_host_bridge_uport().
+ */
+ if (ctx.cxl_version == ACPI_CEDT_CHBS_VERSION_CXL11) {
+ dev_dbg(match, "RCRB found for UID %lld: %pa\n", ctx.uid,
+ &ctx.base);
+ dport = devm_cxl_add_rch_dport(root_port, bridge, ctx.uid,
+ ctx.base);
+ } else {
+ dport = devm_cxl_add_dport(root_port, bridge, ctx.uid,
+ CXL_RESOURCE_NONE);
+ }
+
+ if (IS_ERR(dport))
+ return PTR_ERR(dport);
+
+ return 0;
+}
+
/*
* A host bridge is a dport to a CFMWS decode and it is a uport to the
* dport (PCIe Root Ports) in the host bridge.
@@ -340,6 +454,8 @@ static int add_host_bridge_uport(struct device *match, void *arg)
struct cxl_dport *dport;
struct cxl_port *port;
struct device *bridge;
+ struct cxl_chbs_context ctx;
+ resource_size_t component_reg_phys;
int rc;
if (!hb)
@@ -358,12 +474,26 @@ static int add_host_bridge_uport(struct device *match, void *arg)
return 0;
}
+ rc = cxl_get_chbs(match, hb, &ctx);
+ if (rc)
+ return rc;
+
+ if (ctx.cxl_version == ACPI_CEDT_CHBS_VERSION_CXL11) {
+ dev_warn(bridge,
+ "CXL CHBS version mismatch, skip port registration\n");
+ return 0;
+ }
+
+ component_reg_phys = ctx.base;
+ if (component_reg_phys != CXL_RESOURCE_NONE)
+ dev_dbg(match, "CHBCR found for UID %lld: %pa\n",
+ ctx.uid, &component_reg_phys);
+
rc = devm_cxl_register_pci_bus(host, bridge, pci_root->bus);
if (rc)
return rc;
- port = devm_cxl_add_port(host, bridge, dport->component_reg_phys,
- dport);
+ port = devm_cxl_add_port(host, bridge, component_reg_phys, dport);
if (IS_ERR(port))
return PTR_ERR(port);
@@ -372,110 +502,6 @@ static int add_host_bridge_uport(struct device *match, void *arg)
return 0;
}
-struct cxl_chbs_context {
- struct device *dev;
- unsigned long long uid;
- resource_size_t rcrb;
- resource_size_t chbcr;
- u32 cxl_version;
-};
-
-static int cxl_get_chbcr(union acpi_subtable_headers *header, void *arg,
- const unsigned long end)
-{
- struct cxl_chbs_context *ctx = arg;
- struct acpi_cedt_chbs *chbs;
-
- if (ctx->chbcr)
- return 0;
-
- chbs = (struct acpi_cedt_chbs *) header;
-
- if (ctx->uid != chbs->uid)
- return 0;
-
- ctx->cxl_version = chbs->cxl_version;
- ctx->rcrb = CXL_RESOURCE_NONE;
- ctx->chbcr = CXL_RESOURCE_NONE;
-
- if (!chbs->base)
- return 0;
-
- if (chbs->cxl_version != ACPI_CEDT_CHBS_VERSION_CXL11) {
- ctx->chbcr = chbs->base;
- return 0;
- }
-
- if (chbs->length != CXL_RCRB_SIZE)
- return 0;
-
- ctx->rcrb = chbs->base;
- ctx->chbcr = cxl_rcrb_to_component(ctx->dev, chbs->base,
- CXL_RCRB_DOWNSTREAM);
-
- return 0;
-}
-
-static int add_host_bridge_dport(struct device *match, void *arg)
-{
- acpi_status rc;
- struct device *bridge;
- unsigned long long uid;
- struct cxl_dport *dport;
- struct cxl_chbs_context ctx;
- struct acpi_pci_root *pci_root;
- struct cxl_port *root_port = arg;
- struct device *host = root_port->dev.parent;
- struct acpi_device *hb = to_cxl_host_bridge(host, match);
-
- if (!hb)
- return 0;
-
- rc = acpi_evaluate_integer(hb->handle, METHOD_NAME__UID, NULL, &uid);
- if (rc != AE_OK) {
- dev_err(match, "unable to retrieve _UID\n");
- return -ENODEV;
- }
-
- dev_dbg(match, "UID found: %lld\n", uid);
-
- ctx = (struct cxl_chbs_context) {
- .dev = match,
- .uid = uid,
- };
- acpi_table_parse_cedt(ACPI_CEDT_TYPE_CHBS, cxl_get_chbcr, &ctx);
-
- if (!ctx.chbcr) {
- dev_warn(match, "No CHBS found for Host Bridge (UID %lld)\n",
- uid);
- return 0;
- }
-
- if (ctx.rcrb != CXL_RESOURCE_NONE)
- dev_dbg(match, "RCRB found for UID %lld: %pa\n", uid, &ctx.rcrb);
-
- if (ctx.chbcr == CXL_RESOURCE_NONE) {
- dev_warn(match, "CHBCR invalid for Host Bridge (UID %lld)\n",
- uid);
- return 0;
- }
-
- dev_dbg(match, "CHBCR found: %pa\n", &ctx.chbcr);
-
- pci_root = acpi_pci_find_root(hb->handle);
- bridge = pci_root->bus->bridge;
- if (ctx.cxl_version == ACPI_CEDT_CHBS_VERSION_CXL11)
- dport = devm_cxl_add_rch_dport(root_port, bridge, uid,
- ctx.chbcr, ctx.rcrb);
- else
- dport = devm_cxl_add_dport(root_port, bridge, uid,
- ctx.chbcr);
- if (IS_ERR(dport))
- return PTR_ERR(dport);
-
- return 0;
-}
-
static int add_root_nvdimm_bridge(struct device *match, void *data)
{
struct cxl_decoder *cxld;
diff --git a/drivers/cxl/core/Makefile b/drivers/cxl/core/Makefile
index ca4ae31..1f66b5d 100644
--- a/drivers/cxl/core/Makefile
+++ b/drivers/cxl/core/Makefile
@@ -12,5 +12,6 @@
cxl_core-y += mbox.o
cxl_core-y += pci.o
cxl_core-y += hdm.o
+cxl_core-y += pmu.o
cxl_core-$(CONFIG_TRACING) += trace.o
cxl_core-$(CONFIG_CXL_REGION) += region.o
diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h
index 27f0968..45e7e04 100644
--- a/drivers/cxl/core/core.h
+++ b/drivers/cxl/core/core.h
@@ -6,6 +6,7 @@
extern const struct device_type cxl_nvdimm_bridge_type;
extern const struct device_type cxl_nvdimm_type;
+extern const struct device_type cxl_pmu_type;
extern struct attribute_group cxl_base_attribute_group;
@@ -63,6 +64,16 @@ int cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, unsigned long long size);
int cxl_dpa_free(struct cxl_endpoint_decoder *cxled);
resource_size_t cxl_dpa_size(struct cxl_endpoint_decoder *cxled);
resource_size_t cxl_dpa_resource_start(struct cxl_endpoint_decoder *cxled);
+
+enum cxl_rcrb {
+ CXL_RCRB_DOWNSTREAM,
+ CXL_RCRB_UPSTREAM,
+};
+struct cxl_rcrb_info;
+resource_size_t __rcrb_to_component(struct device *dev,
+ struct cxl_rcrb_info *ri,
+ enum cxl_rcrb which);
+
extern struct rw_semaphore cxl_dpa_rwsem;
int cxl_memdev_init(void);
diff --git a/drivers/cxl/core/hdm.c b/drivers/cxl/core/hdm.c
index 7889ff2..4449b34 100644
--- a/drivers/cxl/core/hdm.c
+++ b/drivers/cxl/core/hdm.c
@@ -85,6 +85,7 @@ static int map_hdm_decoder_regs(struct cxl_port *port, void __iomem *crb,
struct cxl_component_regs *regs)
{
struct cxl_register_map map = {
+ .dev = &port->dev,
.resource = port->component_reg_phys,
.base = crb,
.max_size = CXL_COMPONENT_REG_BLOCK_SIZE,
@@ -97,8 +98,7 @@ static int map_hdm_decoder_regs(struct cxl_port *port, void __iomem *crb,
return -ENODEV;
}
- return cxl_map_component_regs(&port->dev, regs, &map,
- BIT(CXL_CM_CAP_CAP_ID_HDM));
+ return cxl_map_component_regs(&map, regs, BIT(CXL_CM_CAP_CAP_ID_HDM));
}
static bool should_emulate_decoders(struct cxl_endpoint_dvsec_info *info)
@@ -570,8 +570,9 @@ static void cxld_set_interleave(struct cxl_decoder *cxld, u32 *ctrl)
static void cxld_set_type(struct cxl_decoder *cxld, u32 *ctrl)
{
- u32p_replace_bits(ctrl, !!(cxld->target_type == 3),
- CXL_HDM_DECODER0_CTRL_TYPE);
+ u32p_replace_bits(ctrl,
+ !!(cxld->target_type == CXL_DECODER_HOSTONLYMEM),
+ CXL_HDM_DECODER0_CTRL_HOSTONLY);
}
static int cxlsd_set_targets(struct cxl_switch_decoder *cxlsd, u64 *tgt)
@@ -764,7 +765,7 @@ static int cxl_setup_hdm_decoder_from_dvsec(
if (!len)
return -ENOENT;
- cxld->target_type = CXL_DECODER_EXPANDER;
+ cxld->target_type = CXL_DECODER_HOSTONLYMEM;
cxld->commit = NULL;
cxld->reset = NULL;
cxld->hpa_range = info->dvsec_range[which];
@@ -793,8 +794,8 @@ static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
int *target_map, void __iomem *hdm, int which,
u64 *dpa_base, struct cxl_endpoint_dvsec_info *info)
{
+ struct cxl_endpoint_decoder *cxled = NULL;
u64 size, base, skip, dpa_size, lo, hi;
- struct cxl_endpoint_decoder *cxled;
bool committed;
u32 remainder;
int i, rc;
@@ -827,6 +828,8 @@ static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
return -ENXIO;
}
+ if (info)
+ cxled = to_cxl_endpoint_decoder(&cxld->dev);
cxld->hpa_range = (struct range) {
.start = base,
.end = base + size - 1,
@@ -837,10 +840,10 @@ static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
cxld->flags |= CXL_DECODER_F_ENABLE;
if (ctrl & CXL_HDM_DECODER0_CTRL_LOCK)
cxld->flags |= CXL_DECODER_F_LOCK;
- if (FIELD_GET(CXL_HDM_DECODER0_CTRL_TYPE, ctrl))
- cxld->target_type = CXL_DECODER_EXPANDER;
+ if (FIELD_GET(CXL_HDM_DECODER0_CTRL_HOSTONLY, ctrl))
+ cxld->target_type = CXL_DECODER_HOSTONLYMEM;
else
- cxld->target_type = CXL_DECODER_ACCELERATOR;
+ cxld->target_type = CXL_DECODER_DEVMEM;
if (cxld->id != port->commit_end + 1) {
dev_warn(&port->dev,
"decoder%d.%d: Committed out of order\n",
@@ -856,12 +859,28 @@ static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
}
port->commit_end = cxld->id;
} else {
- /* unless / until type-2 drivers arrive, assume type-3 */
- if (FIELD_GET(CXL_HDM_DECODER0_CTRL_TYPE, ctrl) == 0) {
- ctrl |= CXL_HDM_DECODER0_CTRL_TYPE;
+ if (cxled) {
+ struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
+ struct cxl_dev_state *cxlds = cxlmd->cxlds;
+
+ /*
+ * Default by devtype until a device arrives that needs
+ * more precision.
+ */
+ if (cxlds->type == CXL_DEVTYPE_CLASSMEM)
+ cxld->target_type = CXL_DECODER_HOSTONLYMEM;
+ else
+ cxld->target_type = CXL_DECODER_DEVMEM;
+ } else {
+ /* To be overridden by region type at commit time */
+ cxld->target_type = CXL_DECODER_HOSTONLYMEM;
+ }
+
+ if (!FIELD_GET(CXL_HDM_DECODER0_CTRL_HOSTONLY, ctrl) &&
+ cxld->target_type == CXL_DECODER_HOSTONLYMEM) {
+ ctrl |= CXL_HDM_DECODER0_CTRL_HOSTONLY;
writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which));
}
- cxld->target_type = CXL_DECODER_EXPANDER;
}
rc = eiw_to_ways(FIELD_GET(CXL_HDM_DECODER0_CTRL_IW_MASK, ctrl),
&cxld->interleave_ways);
@@ -880,7 +899,7 @@ static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
port->id, cxld->id, cxld->hpa_range.start, cxld->hpa_range.end,
cxld->interleave_ways, cxld->interleave_granularity);
- if (!info) {
+ if (!cxled) {
lo = readl(hdm + CXL_HDM_DECODER0_TL_LOW(which));
hi = readl(hdm + CXL_HDM_DECODER0_TL_HIGH(which));
target_list.value = (hi << 32) + lo;
@@ -903,7 +922,6 @@ static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
lo = readl(hdm + CXL_HDM_DECODER0_SKIP_LOW(which));
hi = readl(hdm + CXL_HDM_DECODER0_SKIP_HIGH(which));
skip = (hi << 32) + lo;
- cxled = to_cxl_endpoint_decoder(&cxld->dev);
rc = devm_cxl_dpa_reserve(cxled, *dpa_base + skip, dpa_size, skip);
if (rc) {
dev_err(&port->dev,
diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c
index bea9cf3..d6d067f 100644
--- a/drivers/cxl/core/mbox.c
+++ b/drivers/cxl/core/mbox.c
@@ -182,7 +182,7 @@ static const char *cxl_mem_opcode_to_name(u16 opcode)
/**
* cxl_internal_send_cmd() - Kernel internal interface to send a mailbox command
- * @cxlds: The device data for the operation
+ * @mds: The driver data for the operation
* @mbox_cmd: initialized command to execute
*
* Context: Any context.
@@ -198,19 +198,19 @@ static const char *cxl_mem_opcode_to_name(u16 opcode)
* error. While this distinction can be useful for commands from userspace, the
* kernel will only be able to use results when both are successful.
*/
-int cxl_internal_send_cmd(struct cxl_dev_state *cxlds,
+int cxl_internal_send_cmd(struct cxl_memdev_state *mds,
struct cxl_mbox_cmd *mbox_cmd)
{
size_t out_size, min_out;
int rc;
- if (mbox_cmd->size_in > cxlds->payload_size ||
- mbox_cmd->size_out > cxlds->payload_size)
+ if (mbox_cmd->size_in > mds->payload_size ||
+ mbox_cmd->size_out > mds->payload_size)
return -E2BIG;
out_size = mbox_cmd->size_out;
min_out = mbox_cmd->min_out;
- rc = cxlds->mbox_send(cxlds, mbox_cmd);
+ rc = mds->mbox_send(mds, mbox_cmd);
/*
* EIO is reserved for a payload size mismatch and mbox_send()
* may not return this error.
@@ -220,7 +220,8 @@ int cxl_internal_send_cmd(struct cxl_dev_state *cxlds,
if (rc)
return rc;
- if (mbox_cmd->return_code != CXL_MBOX_CMD_RC_SUCCESS)
+ if (mbox_cmd->return_code != CXL_MBOX_CMD_RC_SUCCESS &&
+ mbox_cmd->return_code != CXL_MBOX_CMD_RC_BACKGROUND)
return cxl_mbox_cmd_rc2errno(mbox_cmd);
if (!out_size)
@@ -297,7 +298,7 @@ static bool cxl_payload_from_user_allowed(u16 opcode, void *payload_in)
}
static int cxl_mbox_cmd_ctor(struct cxl_mbox_cmd *mbox,
- struct cxl_dev_state *cxlds, u16 opcode,
+ struct cxl_memdev_state *mds, u16 opcode,
size_t in_size, size_t out_size, u64 in_payload)
{
*mbox = (struct cxl_mbox_cmd) {
@@ -312,7 +313,7 @@ static int cxl_mbox_cmd_ctor(struct cxl_mbox_cmd *mbox,
return PTR_ERR(mbox->payload_in);
if (!cxl_payload_from_user_allowed(opcode, mbox->payload_in)) {
- dev_dbg(cxlds->dev, "%s: input payload not allowed\n",
+ dev_dbg(mds->cxlds.dev, "%s: input payload not allowed\n",
cxl_mem_opcode_to_name(opcode));
kvfree(mbox->payload_in);
return -EBUSY;
@@ -321,7 +322,7 @@ static int cxl_mbox_cmd_ctor(struct cxl_mbox_cmd *mbox,
/* Prepare to handle a full payload for variable sized output */
if (out_size == CXL_VARIABLE_PAYLOAD)
- mbox->size_out = cxlds->payload_size;
+ mbox->size_out = mds->payload_size;
else
mbox->size_out = out_size;
@@ -343,7 +344,7 @@ static void cxl_mbox_cmd_dtor(struct cxl_mbox_cmd *mbox)
static int cxl_to_mem_cmd_raw(struct cxl_mem_command *mem_cmd,
const struct cxl_send_command *send_cmd,
- struct cxl_dev_state *cxlds)
+ struct cxl_memdev_state *mds)
{
if (send_cmd->raw.rsvd)
return -EINVAL;
@@ -353,13 +354,13 @@ static int cxl_to_mem_cmd_raw(struct cxl_mem_command *mem_cmd,
* gets passed along without further checking, so it must be
* validated here.
*/
- if (send_cmd->out.size > cxlds->payload_size)
+ if (send_cmd->out.size > mds->payload_size)
return -EINVAL;
if (!cxl_mem_raw_command_allowed(send_cmd->raw.opcode))
return -EPERM;
- dev_WARN_ONCE(cxlds->dev, true, "raw command path used\n");
+ dev_WARN_ONCE(mds->cxlds.dev, true, "raw command path used\n");
*mem_cmd = (struct cxl_mem_command) {
.info = {
@@ -375,7 +376,7 @@ static int cxl_to_mem_cmd_raw(struct cxl_mem_command *mem_cmd,
static int cxl_to_mem_cmd(struct cxl_mem_command *mem_cmd,
const struct cxl_send_command *send_cmd,
- struct cxl_dev_state *cxlds)
+ struct cxl_memdev_state *mds)
{
struct cxl_mem_command *c = &cxl_mem_commands[send_cmd->id];
const struct cxl_command_info *info = &c->info;
@@ -390,11 +391,11 @@ static int cxl_to_mem_cmd(struct cxl_mem_command *mem_cmd,
return -EINVAL;
/* Check that the command is enabled for hardware */
- if (!test_bit(info->id, cxlds->enabled_cmds))
+ if (!test_bit(info->id, mds->enabled_cmds))
return -ENOTTY;
/* Check that the command is not claimed for exclusive kernel use */
- if (test_bit(info->id, cxlds->exclusive_cmds))
+ if (test_bit(info->id, mds->exclusive_cmds))
return -EBUSY;
/* Check the input buffer is the expected size */
@@ -423,7 +424,7 @@ static int cxl_to_mem_cmd(struct cxl_mem_command *mem_cmd,
/**
* cxl_validate_cmd_from_user() - Check fields for CXL_MEM_SEND_COMMAND.
* @mbox_cmd: Sanitized and populated &struct cxl_mbox_cmd.
- * @cxlds: The device data for the operation
+ * @mds: The driver data for the operation
* @send_cmd: &struct cxl_send_command copied in from userspace.
*
* Return:
@@ -438,7 +439,7 @@ static int cxl_to_mem_cmd(struct cxl_mem_command *mem_cmd,
* safe to send to the hardware.
*/
static int cxl_validate_cmd_from_user(struct cxl_mbox_cmd *mbox_cmd,
- struct cxl_dev_state *cxlds,
+ struct cxl_memdev_state *mds,
const struct cxl_send_command *send_cmd)
{
struct cxl_mem_command mem_cmd;
@@ -452,20 +453,20 @@ static int cxl_validate_cmd_from_user(struct cxl_mbox_cmd *mbox_cmd,
* supports, but output can be arbitrarily large (simply write out as
* much data as the hardware provides).
*/
- if (send_cmd->in.size > cxlds->payload_size)
+ if (send_cmd->in.size > mds->payload_size)
return -EINVAL;
/* Sanitize and construct a cxl_mem_command */
if (send_cmd->id == CXL_MEM_COMMAND_ID_RAW)
- rc = cxl_to_mem_cmd_raw(&mem_cmd, send_cmd, cxlds);
+ rc = cxl_to_mem_cmd_raw(&mem_cmd, send_cmd, mds);
else
- rc = cxl_to_mem_cmd(&mem_cmd, send_cmd, cxlds);
+ rc = cxl_to_mem_cmd(&mem_cmd, send_cmd, mds);
if (rc)
return rc;
/* Sanitize and construct a cxl_mbox_cmd */
- return cxl_mbox_cmd_ctor(mbox_cmd, cxlds, mem_cmd.opcode,
+ return cxl_mbox_cmd_ctor(mbox_cmd, mds, mem_cmd.opcode,
mem_cmd.info.size_in, mem_cmd.info.size_out,
send_cmd->in.payload);
}
@@ -473,6 +474,7 @@ static int cxl_validate_cmd_from_user(struct cxl_mbox_cmd *mbox_cmd,
int cxl_query_cmd(struct cxl_memdev *cxlmd,
struct cxl_mem_query_commands __user *q)
{
+ struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
struct device *dev = &cxlmd->dev;
struct cxl_mem_command *cmd;
u32 n_commands;
@@ -494,9 +496,9 @@ int cxl_query_cmd(struct cxl_memdev *cxlmd,
cxl_for_each_cmd(cmd) {
struct cxl_command_info info = cmd->info;
- if (test_bit(info.id, cxlmd->cxlds->enabled_cmds))
+ if (test_bit(info.id, mds->enabled_cmds))
info.flags |= CXL_MEM_COMMAND_FLAG_ENABLED;
- if (test_bit(info.id, cxlmd->cxlds->exclusive_cmds))
+ if (test_bit(info.id, mds->exclusive_cmds))
info.flags |= CXL_MEM_COMMAND_FLAG_EXCLUSIVE;
if (copy_to_user(&q->commands[j++], &info, sizeof(info)))
@@ -511,7 +513,7 @@ int cxl_query_cmd(struct cxl_memdev *cxlmd,
/**
* handle_mailbox_cmd_from_user() - Dispatch a mailbox command for userspace.
- * @cxlds: The device data for the operation
+ * @mds: The driver data for the operation
* @mbox_cmd: The validated mailbox command.
* @out_payload: Pointer to userspace's output payload.
* @size_out: (Input) Max payload size to copy out.
@@ -532,12 +534,12 @@ int cxl_query_cmd(struct cxl_memdev *cxlmd,
*
* See cxl_send_cmd().
*/
-static int handle_mailbox_cmd_from_user(struct cxl_dev_state *cxlds,
+static int handle_mailbox_cmd_from_user(struct cxl_memdev_state *mds,
struct cxl_mbox_cmd *mbox_cmd,
u64 out_payload, s32 *size_out,
u32 *retval)
{
- struct device *dev = cxlds->dev;
+ struct device *dev = mds->cxlds.dev;
int rc;
dev_dbg(dev,
@@ -547,7 +549,7 @@ static int handle_mailbox_cmd_from_user(struct cxl_dev_state *cxlds,
cxl_mem_opcode_to_name(mbox_cmd->opcode),
mbox_cmd->opcode, mbox_cmd->size_in);
- rc = cxlds->mbox_send(cxlds, mbox_cmd);
+ rc = mds->mbox_send(mds, mbox_cmd);
if (rc)
goto out;
@@ -576,7 +578,7 @@ static int handle_mailbox_cmd_from_user(struct cxl_dev_state *cxlds,
int cxl_send_cmd(struct cxl_memdev *cxlmd, struct cxl_send_command __user *s)
{
- struct cxl_dev_state *cxlds = cxlmd->cxlds;
+ struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
struct device *dev = &cxlmd->dev;
struct cxl_send_command send;
struct cxl_mbox_cmd mbox_cmd;
@@ -587,11 +589,11 @@ int cxl_send_cmd(struct cxl_memdev *cxlmd, struct cxl_send_command __user *s)
if (copy_from_user(&send, s, sizeof(send)))
return -EFAULT;
- rc = cxl_validate_cmd_from_user(&mbox_cmd, cxlmd->cxlds, &send);
+ rc = cxl_validate_cmd_from_user(&mbox_cmd, mds, &send);
if (rc)
return rc;
- rc = handle_mailbox_cmd_from_user(cxlds, &mbox_cmd, send.out.payload,
+ rc = handle_mailbox_cmd_from_user(mds, &mbox_cmd, send.out.payload,
&send.out.size, &send.retval);
if (rc)
return rc;
@@ -602,13 +604,14 @@ int cxl_send_cmd(struct cxl_memdev *cxlmd, struct cxl_send_command __user *s)
return 0;
}
-static int cxl_xfer_log(struct cxl_dev_state *cxlds, uuid_t *uuid, u32 *size, u8 *out)
+static int cxl_xfer_log(struct cxl_memdev_state *mds, uuid_t *uuid,
+ u32 *size, u8 *out)
{
u32 remaining = *size;
u32 offset = 0;
while (remaining) {
- u32 xfer_size = min_t(u32, remaining, cxlds->payload_size);
+ u32 xfer_size = min_t(u32, remaining, mds->payload_size);
struct cxl_mbox_cmd mbox_cmd;
struct cxl_mbox_get_log log;
int rc;
@@ -627,7 +630,7 @@ static int cxl_xfer_log(struct cxl_dev_state *cxlds, uuid_t *uuid, u32 *size, u8
.payload_out = out,
};
- rc = cxl_internal_send_cmd(cxlds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(mds, &mbox_cmd);
/*
* The output payload length that indicates the number
@@ -654,17 +657,18 @@ static int cxl_xfer_log(struct cxl_dev_state *cxlds, uuid_t *uuid, u32 *size, u8
/**
* cxl_walk_cel() - Walk through the Command Effects Log.
- * @cxlds: The device data for the operation
+ * @mds: The driver data for the operation
* @size: Length of the Command Effects Log.
* @cel: CEL
*
* Iterate over each entry in the CEL and determine if the driver supports the
* command. If so, the command is enabled for the device and can be used later.
*/
-static void cxl_walk_cel(struct cxl_dev_state *cxlds, size_t size, u8 *cel)
+static void cxl_walk_cel(struct cxl_memdev_state *mds, size_t size, u8 *cel)
{
struct cxl_cel_entry *cel_entry;
const int cel_entries = size / sizeof(*cel_entry);
+ struct device *dev = mds->cxlds.dev;
int i;
cel_entry = (struct cxl_cel_entry *) cel;
@@ -674,39 +678,39 @@ static void cxl_walk_cel(struct cxl_dev_state *cxlds, size_t size, u8 *cel)
struct cxl_mem_command *cmd = cxl_mem_find_command(opcode);
if (!cmd && !cxl_is_poison_command(opcode)) {
- dev_dbg(cxlds->dev,
+ dev_dbg(dev,
"Opcode 0x%04x unsupported by driver\n", opcode);
continue;
}
if (cmd)
- set_bit(cmd->info.id, cxlds->enabled_cmds);
+ set_bit(cmd->info.id, mds->enabled_cmds);
if (cxl_is_poison_command(opcode))
- cxl_set_poison_cmd_enabled(&cxlds->poison, opcode);
+ cxl_set_poison_cmd_enabled(&mds->poison, opcode);
- dev_dbg(cxlds->dev, "Opcode 0x%04x enabled\n", opcode);
+ dev_dbg(dev, "Opcode 0x%04x enabled\n", opcode);
}
}
-static struct cxl_mbox_get_supported_logs *cxl_get_gsl(struct cxl_dev_state *cxlds)
+static struct cxl_mbox_get_supported_logs *cxl_get_gsl(struct cxl_memdev_state *mds)
{
struct cxl_mbox_get_supported_logs *ret;
struct cxl_mbox_cmd mbox_cmd;
int rc;
- ret = kvmalloc(cxlds->payload_size, GFP_KERNEL);
+ ret = kvmalloc(mds->payload_size, GFP_KERNEL);
if (!ret)
return ERR_PTR(-ENOMEM);
mbox_cmd = (struct cxl_mbox_cmd) {
.opcode = CXL_MBOX_OP_GET_SUPPORTED_LOGS,
- .size_out = cxlds->payload_size,
+ .size_out = mds->payload_size,
.payload_out = ret,
/* At least the record number field must be valid */
.min_out = 2,
};
- rc = cxl_internal_send_cmd(cxlds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(mds, &mbox_cmd);
if (rc < 0) {
kvfree(ret);
return ERR_PTR(rc);
@@ -729,22 +733,22 @@ static const uuid_t log_uuid[] = {
/**
* cxl_enumerate_cmds() - Enumerate commands for a device.
- * @cxlds: The device data for the operation
+ * @mds: The driver data for the operation
*
* Returns 0 if enumerate completed successfully.
*
* CXL devices have optional support for certain commands. This function will
* determine the set of supported commands for the hardware and update the
- * enabled_cmds bitmap in the @cxlds.
+ * enabled_cmds bitmap in the @mds.
*/
-int cxl_enumerate_cmds(struct cxl_dev_state *cxlds)
+int cxl_enumerate_cmds(struct cxl_memdev_state *mds)
{
struct cxl_mbox_get_supported_logs *gsl;
- struct device *dev = cxlds->dev;
+ struct device *dev = mds->cxlds.dev;
struct cxl_mem_command *cmd;
int i, rc;
- gsl = cxl_get_gsl(cxlds);
+ gsl = cxl_get_gsl(mds);
if (IS_ERR(gsl))
return PTR_ERR(gsl);
@@ -765,19 +769,19 @@ int cxl_enumerate_cmds(struct cxl_dev_state *cxlds)
goto out;
}
- rc = cxl_xfer_log(cxlds, &uuid, &size, log);
+ rc = cxl_xfer_log(mds, &uuid, &size, log);
if (rc) {
kvfree(log);
goto out;
}
- cxl_walk_cel(cxlds, size, log);
+ cxl_walk_cel(mds, size, log);
kvfree(log);
/* In case CEL was bogus, enable some default commands. */
cxl_for_each_cmd(cmd)
if (cmd->flags & CXL_CMD_FLAG_FORCE_ENABLE)
- set_bit(cmd->info.id, cxlds->enabled_cmds);
+ set_bit(cmd->info.id, mds->enabled_cmds);
/* Found the required CEL */
rc = 0;
@@ -838,7 +842,7 @@ static void cxl_event_trace_record(const struct cxl_memdev *cxlmd,
}
}
-static int cxl_clear_event_record(struct cxl_dev_state *cxlds,
+static int cxl_clear_event_record(struct cxl_memdev_state *mds,
enum cxl_event_log_type log,
struct cxl_get_event_payload *get_pl)
{
@@ -852,9 +856,9 @@ static int cxl_clear_event_record(struct cxl_dev_state *cxlds,
int i;
/* Payload size may limit the max handles */
- if (pl_size > cxlds->payload_size) {
- max_handles = (cxlds->payload_size - sizeof(*payload)) /
- sizeof(__le16);
+ if (pl_size > mds->payload_size) {
+ max_handles = (mds->payload_size - sizeof(*payload)) /
+ sizeof(__le16);
pl_size = struct_size(payload, handles, max_handles);
}
@@ -879,12 +883,12 @@ static int cxl_clear_event_record(struct cxl_dev_state *cxlds,
i = 0;
for (cnt = 0; cnt < total; cnt++) {
payload->handles[i++] = get_pl->records[cnt].hdr.handle;
- dev_dbg(cxlds->dev, "Event log '%d': Clearing %u\n",
- log, le16_to_cpu(payload->handles[i]));
+ dev_dbg(mds->cxlds.dev, "Event log '%d': Clearing %u\n", log,
+ le16_to_cpu(payload->handles[i]));
if (i == max_handles) {
payload->nr_recs = i;
- rc = cxl_internal_send_cmd(cxlds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(mds, &mbox_cmd);
if (rc)
goto free_pl;
i = 0;
@@ -895,7 +899,7 @@ static int cxl_clear_event_record(struct cxl_dev_state *cxlds,
if (i) {
payload->nr_recs = i;
mbox_cmd.size_in = struct_size(payload, handles, i);
- rc = cxl_internal_send_cmd(cxlds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(mds, &mbox_cmd);
if (rc)
goto free_pl;
}
@@ -905,32 +909,34 @@ static int cxl_clear_event_record(struct cxl_dev_state *cxlds,
return rc;
}
-static void cxl_mem_get_records_log(struct cxl_dev_state *cxlds,
+static void cxl_mem_get_records_log(struct cxl_memdev_state *mds,
enum cxl_event_log_type type)
{
+ struct cxl_memdev *cxlmd = mds->cxlds.cxlmd;
+ struct device *dev = mds->cxlds.dev;
struct cxl_get_event_payload *payload;
struct cxl_mbox_cmd mbox_cmd;
u8 log_type = type;
u16 nr_rec;
- mutex_lock(&cxlds->event.log_lock);
- payload = cxlds->event.buf;
+ mutex_lock(&mds->event.log_lock);
+ payload = mds->event.buf;
mbox_cmd = (struct cxl_mbox_cmd) {
.opcode = CXL_MBOX_OP_GET_EVENT_RECORD,
.payload_in = &log_type,
.size_in = sizeof(log_type),
.payload_out = payload,
- .size_out = cxlds->payload_size,
+ .size_out = mds->payload_size,
.min_out = struct_size(payload, records, 0),
};
do {
int rc, i;
- rc = cxl_internal_send_cmd(cxlds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(mds, &mbox_cmd);
if (rc) {
- dev_err_ratelimited(cxlds->dev,
+ dev_err_ratelimited(dev,
"Event log '%d': Failed to query event records : %d",
type, rc);
break;
@@ -941,27 +947,27 @@ static void cxl_mem_get_records_log(struct cxl_dev_state *cxlds,
break;
for (i = 0; i < nr_rec; i++)
- cxl_event_trace_record(cxlds->cxlmd, type,
+ cxl_event_trace_record(cxlmd, type,
&payload->records[i]);
if (payload->flags & CXL_GET_EVENT_FLAG_OVERFLOW)
- trace_cxl_overflow(cxlds->cxlmd, type, payload);
+ trace_cxl_overflow(cxlmd, type, payload);
- rc = cxl_clear_event_record(cxlds, type, payload);
+ rc = cxl_clear_event_record(mds, type, payload);
if (rc) {
- dev_err_ratelimited(cxlds->dev,
+ dev_err_ratelimited(dev,
"Event log '%d': Failed to clear events : %d",
type, rc);
break;
}
} while (nr_rec);
- mutex_unlock(&cxlds->event.log_lock);
+ mutex_unlock(&mds->event.log_lock);
}
/**
* cxl_mem_get_event_records - Get Event Records from the device
- * @cxlds: The device data for the operation
+ * @mds: The driver data for the operation
* @status: Event Status register value identifying which events are available.
*
* Retrieve all event records available on the device, report them as trace
@@ -970,24 +976,24 @@ static void cxl_mem_get_records_log(struct cxl_dev_state *cxlds,
* See CXL rev 3.0 @8.2.9.2.2 Get Event Records
* See CXL rev 3.0 @8.2.9.2.3 Clear Event Records
*/
-void cxl_mem_get_event_records(struct cxl_dev_state *cxlds, u32 status)
+void cxl_mem_get_event_records(struct cxl_memdev_state *mds, u32 status)
{
- dev_dbg(cxlds->dev, "Reading event logs: %x\n", status);
+ dev_dbg(mds->cxlds.dev, "Reading event logs: %x\n", status);
if (status & CXLDEV_EVENT_STATUS_FATAL)
- cxl_mem_get_records_log(cxlds, CXL_EVENT_TYPE_FATAL);
+ cxl_mem_get_records_log(mds, CXL_EVENT_TYPE_FATAL);
if (status & CXLDEV_EVENT_STATUS_FAIL)
- cxl_mem_get_records_log(cxlds, CXL_EVENT_TYPE_FAIL);
+ cxl_mem_get_records_log(mds, CXL_EVENT_TYPE_FAIL);
if (status & CXLDEV_EVENT_STATUS_WARN)
- cxl_mem_get_records_log(cxlds, CXL_EVENT_TYPE_WARN);
+ cxl_mem_get_records_log(mds, CXL_EVENT_TYPE_WARN);
if (status & CXLDEV_EVENT_STATUS_INFO)
- cxl_mem_get_records_log(cxlds, CXL_EVENT_TYPE_INFO);
+ cxl_mem_get_records_log(mds, CXL_EVENT_TYPE_INFO);
}
EXPORT_SYMBOL_NS_GPL(cxl_mem_get_event_records, CXL);
/**
* cxl_mem_get_partition_info - Get partition info
- * @cxlds: The device data for the operation
+ * @mds: The driver data for the operation
*
* Retrieve the current partition info for the device specified. The active
* values are the current capacity in bytes. If not 0, the 'next' values are
@@ -997,7 +1003,7 @@ EXPORT_SYMBOL_NS_GPL(cxl_mem_get_event_records, CXL);
*
* See CXL @8.2.9.5.2.1 Get Partition Info
*/
-static int cxl_mem_get_partition_info(struct cxl_dev_state *cxlds)
+static int cxl_mem_get_partition_info(struct cxl_memdev_state *mds)
{
struct cxl_mbox_get_partition_info pi;
struct cxl_mbox_cmd mbox_cmd;
@@ -1008,17 +1014,17 @@ static int cxl_mem_get_partition_info(struct cxl_dev_state *cxlds)
.size_out = sizeof(pi),
.payload_out = &pi,
};
- rc = cxl_internal_send_cmd(cxlds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(mds, &mbox_cmd);
if (rc)
return rc;
- cxlds->active_volatile_bytes =
+ mds->active_volatile_bytes =
le64_to_cpu(pi.active_volatile_cap) * CXL_CAPACITY_MULTIPLIER;
- cxlds->active_persistent_bytes =
+ mds->active_persistent_bytes =
le64_to_cpu(pi.active_persistent_cap) * CXL_CAPACITY_MULTIPLIER;
- cxlds->next_volatile_bytes =
+ mds->next_volatile_bytes =
le64_to_cpu(pi.next_volatile_cap) * CXL_CAPACITY_MULTIPLIER;
- cxlds->next_persistent_bytes =
+ mds->next_persistent_bytes =
le64_to_cpu(pi.next_volatile_cap) * CXL_CAPACITY_MULTIPLIER;
return 0;
@@ -1026,14 +1032,14 @@ static int cxl_mem_get_partition_info(struct cxl_dev_state *cxlds)
/**
* cxl_dev_state_identify() - Send the IDENTIFY command to the device.
- * @cxlds: The device data for the operation
+ * @mds: The driver data for the operation
*
* Return: 0 if identify was executed successfully or media not ready.
*
* This will dispatch the identify command to the device and on success populate
* structures to be exported to sysfs.
*/
-int cxl_dev_state_identify(struct cxl_dev_state *cxlds)
+int cxl_dev_state_identify(struct cxl_memdev_state *mds)
{
/* See CXL 2.0 Table 175 Identify Memory Device Output Payload */
struct cxl_mbox_identify id;
@@ -1041,7 +1047,7 @@ int cxl_dev_state_identify(struct cxl_dev_state *cxlds)
u32 val;
int rc;
- if (!cxlds->media_ready)
+ if (!mds->cxlds.media_ready)
return 0;
mbox_cmd = (struct cxl_mbox_cmd) {
@@ -1049,31 +1055,92 @@ int cxl_dev_state_identify(struct cxl_dev_state *cxlds)
.size_out = sizeof(id),
.payload_out = &id,
};
- rc = cxl_internal_send_cmd(cxlds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(mds, &mbox_cmd);
if (rc < 0)
return rc;
- cxlds->total_bytes =
+ mds->total_bytes =
le64_to_cpu(id.total_capacity) * CXL_CAPACITY_MULTIPLIER;
- cxlds->volatile_only_bytes =
+ mds->volatile_only_bytes =
le64_to_cpu(id.volatile_capacity) * CXL_CAPACITY_MULTIPLIER;
- cxlds->persistent_only_bytes =
+ mds->persistent_only_bytes =
le64_to_cpu(id.persistent_capacity) * CXL_CAPACITY_MULTIPLIER;
- cxlds->partition_align_bytes =
+ mds->partition_align_bytes =
le64_to_cpu(id.partition_align) * CXL_CAPACITY_MULTIPLIER;
- cxlds->lsa_size = le32_to_cpu(id.lsa_size);
- memcpy(cxlds->firmware_version, id.fw_revision, sizeof(id.fw_revision));
+ mds->lsa_size = le32_to_cpu(id.lsa_size);
+ memcpy(mds->firmware_version, id.fw_revision,
+ sizeof(id.fw_revision));
- if (test_bit(CXL_POISON_ENABLED_LIST, cxlds->poison.enabled_cmds)) {
+ if (test_bit(CXL_POISON_ENABLED_LIST, mds->poison.enabled_cmds)) {
val = get_unaligned_le24(id.poison_list_max_mer);
- cxlds->poison.max_errors = min_t(u32, val, CXL_POISON_LIST_MAX);
+ mds->poison.max_errors = min_t(u32, val, CXL_POISON_LIST_MAX);
}
return 0;
}
EXPORT_SYMBOL_NS_GPL(cxl_dev_state_identify, CXL);
+/**
+ * cxl_mem_sanitize() - Send a sanitization command to the device.
+ * @mds: The device data for the operation
+ * @cmd: The specific sanitization command opcode
+ *
+ * Return: 0 if the command was executed successfully, regardless of
+ * whether or not the actual security operation is done in the background,
+ * such as for the Sanitize case.
+ * Error return values can be the result of the mailbox command, -EINVAL
+ * when security requirements are not met or invalid contexts.
+ *
+ * See CXL 3.0 @8.2.9.8.5.1 Sanitize and @8.2.9.8.5.2 Secure Erase.
+ */
+int cxl_mem_sanitize(struct cxl_memdev_state *mds, u16 cmd)
+{
+ int rc;
+ u32 sec_out = 0;
+ struct cxl_get_security_output {
+ __le32 flags;
+ } out;
+ struct cxl_mbox_cmd sec_cmd = {
+ .opcode = CXL_MBOX_OP_GET_SECURITY_STATE,
+ .payload_out = &out,
+ .size_out = sizeof(out),
+ };
+ struct cxl_mbox_cmd mbox_cmd = { .opcode = cmd };
+ struct cxl_dev_state *cxlds = &mds->cxlds;
+
+ if (cmd != CXL_MBOX_OP_SANITIZE && cmd != CXL_MBOX_OP_SECURE_ERASE)
+ return -EINVAL;
+
+ rc = cxl_internal_send_cmd(mds, &sec_cmd);
+ if (rc < 0) {
+ dev_err(cxlds->dev, "Failed to get security state : %d", rc);
+ return rc;
+ }
+
+ /*
+ * Prior to using these commands, any security applied to
+ * the user data areas of the device shall be DISABLED (or
+ * UNLOCKED for secure erase case).
+ */
+ sec_out = le32_to_cpu(out.flags);
+ if (sec_out & CXL_PMEM_SEC_STATE_USER_PASS_SET)
+ return -EINVAL;
+
+ if (cmd == CXL_MBOX_OP_SECURE_ERASE &&
+ sec_out & CXL_PMEM_SEC_STATE_LOCKED)
+ return -EINVAL;
+
+ rc = cxl_internal_send_cmd(mds, &mbox_cmd);
+ if (rc < 0) {
+ dev_err(cxlds->dev, "Failed to sanitize device : %d", rc);
+ return rc;
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL_NS_GPL(cxl_mem_sanitize, CXL);
+
static int add_dpa_res(struct device *dev, struct resource *parent,
struct resource *res, resource_size_t start,
resource_size_t size, const char *type)
@@ -1100,8 +1167,9 @@ static int add_dpa_res(struct device *dev, struct resource *parent,
return 0;
}
-int cxl_mem_create_range_info(struct cxl_dev_state *cxlds)
+int cxl_mem_create_range_info(struct cxl_memdev_state *mds)
{
+ struct cxl_dev_state *cxlds = &mds->cxlds;
struct device *dev = cxlds->dev;
int rc;
@@ -1113,35 +1181,35 @@ int cxl_mem_create_range_info(struct cxl_dev_state *cxlds)
}
cxlds->dpa_res =
- (struct resource)DEFINE_RES_MEM(0, cxlds->total_bytes);
+ (struct resource)DEFINE_RES_MEM(0, mds->total_bytes);
- if (cxlds->partition_align_bytes == 0) {
+ if (mds->partition_align_bytes == 0) {
rc = add_dpa_res(dev, &cxlds->dpa_res, &cxlds->ram_res, 0,
- cxlds->volatile_only_bytes, "ram");
+ mds->volatile_only_bytes, "ram");
if (rc)
return rc;
return add_dpa_res(dev, &cxlds->dpa_res, &cxlds->pmem_res,
- cxlds->volatile_only_bytes,
- cxlds->persistent_only_bytes, "pmem");
+ mds->volatile_only_bytes,
+ mds->persistent_only_bytes, "pmem");
}
- rc = cxl_mem_get_partition_info(cxlds);
+ rc = cxl_mem_get_partition_info(mds);
if (rc) {
dev_err(dev, "Failed to query partition information\n");
return rc;
}
rc = add_dpa_res(dev, &cxlds->dpa_res, &cxlds->ram_res, 0,
- cxlds->active_volatile_bytes, "ram");
+ mds->active_volatile_bytes, "ram");
if (rc)
return rc;
return add_dpa_res(dev, &cxlds->dpa_res, &cxlds->pmem_res,
- cxlds->active_volatile_bytes,
- cxlds->active_persistent_bytes, "pmem");
+ mds->active_volatile_bytes,
+ mds->active_persistent_bytes, "pmem");
}
EXPORT_SYMBOL_NS_GPL(cxl_mem_create_range_info, CXL);
-int cxl_set_timestamp(struct cxl_dev_state *cxlds)
+int cxl_set_timestamp(struct cxl_memdev_state *mds)
{
struct cxl_mbox_cmd mbox_cmd;
struct cxl_mbox_set_timestamp_in pi;
@@ -1154,7 +1222,7 @@ int cxl_set_timestamp(struct cxl_dev_state *cxlds)
.payload_in = &pi,
};
- rc = cxl_internal_send_cmd(cxlds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(mds, &mbox_cmd);
/*
* Command is optional. Devices may have another way of providing
* a timestamp, or may return all 0s in timestamp fields.
@@ -1170,18 +1238,18 @@ EXPORT_SYMBOL_NS_GPL(cxl_set_timestamp, CXL);
int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len,
struct cxl_region *cxlr)
{
- struct cxl_dev_state *cxlds = cxlmd->cxlds;
+ struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
struct cxl_mbox_poison_out *po;
struct cxl_mbox_poison_in pi;
struct cxl_mbox_cmd mbox_cmd;
int nr_records = 0;
int rc;
- rc = mutex_lock_interruptible(&cxlds->poison.lock);
+ rc = mutex_lock_interruptible(&mds->poison.lock);
if (rc)
return rc;
- po = cxlds->poison.list_out;
+ po = mds->poison.list_out;
pi.offset = cpu_to_le64(offset);
pi.length = cpu_to_le64(len / CXL_POISON_LEN_MULT);
@@ -1189,13 +1257,13 @@ int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len,
.opcode = CXL_MBOX_OP_GET_POISON,
.size_in = sizeof(pi),
.payload_in = &pi,
- .size_out = cxlds->payload_size,
+ .size_out = mds->payload_size,
.payload_out = po,
.min_out = struct_size(po, record, 0),
};
do {
- rc = cxl_internal_send_cmd(cxlds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(mds, &mbox_cmd);
if (rc)
break;
@@ -1206,14 +1274,14 @@ int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len,
/* Protect against an uncleared _FLAG_MORE */
nr_records = nr_records + le16_to_cpu(po->count);
- if (nr_records >= cxlds->poison.max_errors) {
+ if (nr_records >= mds->poison.max_errors) {
dev_dbg(&cxlmd->dev, "Max Error Records reached: %d\n",
nr_records);
break;
}
} while (po->flags & CXL_POISON_FLAG_MORE);
- mutex_unlock(&cxlds->poison.lock);
+ mutex_unlock(&mds->poison.lock);
return rc;
}
EXPORT_SYMBOL_NS_GPL(cxl_mem_get_poison, CXL);
@@ -1223,52 +1291,53 @@ static void free_poison_buf(void *buf)
kvfree(buf);
}
-/* Get Poison List output buffer is protected by cxlds->poison.lock */
-static int cxl_poison_alloc_buf(struct cxl_dev_state *cxlds)
+/* Get Poison List output buffer is protected by mds->poison.lock */
+static int cxl_poison_alloc_buf(struct cxl_memdev_state *mds)
{
- cxlds->poison.list_out = kvmalloc(cxlds->payload_size, GFP_KERNEL);
- if (!cxlds->poison.list_out)
+ mds->poison.list_out = kvmalloc(mds->payload_size, GFP_KERNEL);
+ if (!mds->poison.list_out)
return -ENOMEM;
- return devm_add_action_or_reset(cxlds->dev, free_poison_buf,
- cxlds->poison.list_out);
+ return devm_add_action_or_reset(mds->cxlds.dev, free_poison_buf,
+ mds->poison.list_out);
}
-int cxl_poison_state_init(struct cxl_dev_state *cxlds)
+int cxl_poison_state_init(struct cxl_memdev_state *mds)
{
int rc;
- if (!test_bit(CXL_POISON_ENABLED_LIST, cxlds->poison.enabled_cmds))
+ if (!test_bit(CXL_POISON_ENABLED_LIST, mds->poison.enabled_cmds))
return 0;
- rc = cxl_poison_alloc_buf(cxlds);
+ rc = cxl_poison_alloc_buf(mds);
if (rc) {
- clear_bit(CXL_POISON_ENABLED_LIST, cxlds->poison.enabled_cmds);
+ clear_bit(CXL_POISON_ENABLED_LIST, mds->poison.enabled_cmds);
return rc;
}
- mutex_init(&cxlds->poison.lock);
+ mutex_init(&mds->poison.lock);
return 0;
}
EXPORT_SYMBOL_NS_GPL(cxl_poison_state_init, CXL);
-struct cxl_dev_state *cxl_dev_state_create(struct device *dev)
+struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev)
{
- struct cxl_dev_state *cxlds;
+ struct cxl_memdev_state *mds;
- cxlds = devm_kzalloc(dev, sizeof(*cxlds), GFP_KERNEL);
- if (!cxlds) {
+ mds = devm_kzalloc(dev, sizeof(*mds), GFP_KERNEL);
+ if (!mds) {
dev_err(dev, "No memory available\n");
return ERR_PTR(-ENOMEM);
}
- mutex_init(&cxlds->mbox_mutex);
- mutex_init(&cxlds->event.log_lock);
- cxlds->dev = dev;
+ mutex_init(&mds->mbox_mutex);
+ mutex_init(&mds->event.log_lock);
+ mds->cxlds.dev = dev;
+ mds->cxlds.type = CXL_DEVTYPE_CLASSMEM;
- return cxlds;
+ return mds;
}
-EXPORT_SYMBOL_NS_GPL(cxl_dev_state_create, CXL);
+EXPORT_SYMBOL_NS_GPL(cxl_memdev_state_create, CXL);
void __init cxl_mbox_init(void)
{
diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
index 057a432..f99e7ec 100644
--- a/drivers/cxl/core/memdev.c
+++ b/drivers/cxl/core/memdev.c
@@ -1,6 +1,8 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright(c) 2020 Intel Corporation. */
+#include <linux/io-64-nonatomic-lo-hi.h>
+#include <linux/firmware.h>
#include <linux/device.h>
#include <linux/slab.h>
#include <linux/idr.h>
@@ -39,8 +41,11 @@ static ssize_t firmware_version_show(struct device *dev,
{
struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
struct cxl_dev_state *cxlds = cxlmd->cxlds;
+ struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
- return sysfs_emit(buf, "%.16s\n", cxlds->firmware_version);
+ if (!mds)
+ return sysfs_emit(buf, "\n");
+ return sysfs_emit(buf, "%.16s\n", mds->firmware_version);
}
static DEVICE_ATTR_RO(firmware_version);
@@ -49,8 +54,11 @@ static ssize_t payload_max_show(struct device *dev,
{
struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
struct cxl_dev_state *cxlds = cxlmd->cxlds;
+ struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
- return sysfs_emit(buf, "%zu\n", cxlds->payload_size);
+ if (!mds)
+ return sysfs_emit(buf, "\n");
+ return sysfs_emit(buf, "%zu\n", mds->payload_size);
}
static DEVICE_ATTR_RO(payload_max);
@@ -59,8 +67,11 @@ static ssize_t label_storage_size_show(struct device *dev,
{
struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
struct cxl_dev_state *cxlds = cxlmd->cxlds;
+ struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
- return sysfs_emit(buf, "%zu\n", cxlds->lsa_size);
+ if (!mds)
+ return sysfs_emit(buf, "\n");
+ return sysfs_emit(buf, "%zu\n", mds->lsa_size);
}
static DEVICE_ATTR_RO(label_storage_size);
@@ -107,6 +118,89 @@ static ssize_t numa_node_show(struct device *dev, struct device_attribute *attr,
}
static DEVICE_ATTR_RO(numa_node);
+static ssize_t security_state_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
+ struct cxl_dev_state *cxlds = cxlmd->cxlds;
+ struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
+ u64 reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_BG_CMD_STATUS_OFFSET);
+ u32 pct = FIELD_GET(CXLDEV_MBOX_BG_CMD_COMMAND_PCT_MASK, reg);
+ u16 cmd = FIELD_GET(CXLDEV_MBOX_BG_CMD_COMMAND_OPCODE_MASK, reg);
+ unsigned long state = mds->security.state;
+
+ if (cmd == CXL_MBOX_OP_SANITIZE && pct != 100)
+ return sysfs_emit(buf, "sanitize\n");
+
+ if (!(state & CXL_PMEM_SEC_STATE_USER_PASS_SET))
+ return sysfs_emit(buf, "disabled\n");
+ if (state & CXL_PMEM_SEC_STATE_FROZEN ||
+ state & CXL_PMEM_SEC_STATE_MASTER_PLIMIT ||
+ state & CXL_PMEM_SEC_STATE_USER_PLIMIT)
+ return sysfs_emit(buf, "frozen\n");
+ if (state & CXL_PMEM_SEC_STATE_LOCKED)
+ return sysfs_emit(buf, "locked\n");
+ else
+ return sysfs_emit(buf, "unlocked\n");
+}
+static struct device_attribute dev_attr_security_state =
+ __ATTR(state, 0444, security_state_show, NULL);
+
+static ssize_t security_sanitize_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t len)
+{
+ struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
+ struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
+ struct cxl_port *port = cxlmd->endpoint;
+ bool sanitize;
+ ssize_t rc;
+
+ if (kstrtobool(buf, &sanitize) || !sanitize)
+ return -EINVAL;
+
+ if (!port || !is_cxl_endpoint(port))
+ return -EINVAL;
+
+ /* ensure no regions are mapped to this memdev */
+ if (port->commit_end != -1)
+ return -EBUSY;
+
+ rc = cxl_mem_sanitize(mds, CXL_MBOX_OP_SANITIZE);
+
+ return rc ? rc : len;
+}
+static struct device_attribute dev_attr_security_sanitize =
+ __ATTR(sanitize, 0200, NULL, security_sanitize_store);
+
+static ssize_t security_erase_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t len)
+{
+ struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
+ struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
+ struct cxl_port *port = cxlmd->endpoint;
+ ssize_t rc;
+ bool erase;
+
+ if (kstrtobool(buf, &erase) || !erase)
+ return -EINVAL;
+
+ if (!port || !is_cxl_endpoint(port))
+ return -EINVAL;
+
+ /* ensure no regions are mapped to this memdev */
+ if (port->commit_end != -1)
+ return -EBUSY;
+
+ rc = cxl_mem_sanitize(mds, CXL_MBOX_OP_SECURE_ERASE);
+
+ return rc ? rc : len;
+}
+static struct device_attribute dev_attr_security_erase =
+ __ATTR(erase, 0200, NULL, security_erase_store);
+
static int cxl_get_poison_by_memdev(struct cxl_memdev *cxlmd)
{
struct cxl_dev_state *cxlds = cxlmd->cxlds;
@@ -140,7 +234,7 @@ int cxl_trigger_poison_list(struct cxl_memdev *cxlmd)
struct cxl_port *port;
int rc;
- port = dev_get_drvdata(&cxlmd->dev);
+ port = cxlmd->endpoint;
if (!port || !is_cxl_endpoint(port))
return -EINVAL;
@@ -198,7 +292,7 @@ static struct cxl_region *cxl_dpa_to_region(struct cxl_memdev *cxlmd, u64 dpa)
ctx = (struct cxl_dpa_to_region_context) {
.dpa = dpa,
};
- port = dev_get_drvdata(&cxlmd->dev);
+ port = cxlmd->endpoint;
if (port && is_cxl_endpoint(port) && port->commit_end != -1)
device_for_each_child(&port->dev, &ctx, __cxl_dpa_to_region);
@@ -231,7 +325,7 @@ static int cxl_validate_poison_dpa(struct cxl_memdev *cxlmd, u64 dpa)
int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa)
{
- struct cxl_dev_state *cxlds = cxlmd->cxlds;
+ struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
struct cxl_mbox_inject_poison inject;
struct cxl_poison_record record;
struct cxl_mbox_cmd mbox_cmd;
@@ -255,13 +349,13 @@ int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa)
.size_in = sizeof(inject),
.payload_in = &inject,
};
- rc = cxl_internal_send_cmd(cxlds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(mds, &mbox_cmd);
if (rc)
goto out;
cxlr = cxl_dpa_to_region(cxlmd, dpa);
if (cxlr)
- dev_warn_once(cxlds->dev,
+ dev_warn_once(mds->cxlds.dev,
"poison inject dpa:%#llx region: %s\n", dpa,
dev_name(&cxlr->dev));
@@ -279,7 +373,7 @@ EXPORT_SYMBOL_NS_GPL(cxl_inject_poison, CXL);
int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa)
{
- struct cxl_dev_state *cxlds = cxlmd->cxlds;
+ struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
struct cxl_mbox_clear_poison clear;
struct cxl_poison_record record;
struct cxl_mbox_cmd mbox_cmd;
@@ -312,14 +406,15 @@ int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa)
.payload_in = &clear,
};
- rc = cxl_internal_send_cmd(cxlds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(mds, &mbox_cmd);
if (rc)
goto out;
cxlr = cxl_dpa_to_region(cxlmd, dpa);
if (cxlr)
- dev_warn_once(cxlds->dev, "poison clear dpa:%#llx region: %s\n",
- dpa, dev_name(&cxlr->dev));
+ dev_warn_once(mds->cxlds.dev,
+ "poison clear dpa:%#llx region: %s\n", dpa,
+ dev_name(&cxlr->dev));
record = (struct cxl_poison_record) {
.address = cpu_to_le64(dpa),
@@ -352,6 +447,13 @@ static struct attribute *cxl_memdev_ram_attributes[] = {
NULL,
};
+static struct attribute *cxl_memdev_security_attributes[] = {
+ &dev_attr_security_state.attr,
+ &dev_attr_security_sanitize.attr,
+ &dev_attr_security_erase.attr,
+ NULL,
+};
+
static umode_t cxl_memdev_visible(struct kobject *kobj, struct attribute *a,
int n)
{
@@ -375,10 +477,16 @@ static struct attribute_group cxl_memdev_pmem_attribute_group = {
.attrs = cxl_memdev_pmem_attributes,
};
+static struct attribute_group cxl_memdev_security_attribute_group = {
+ .name = "security",
+ .attrs = cxl_memdev_security_attributes,
+};
+
static const struct attribute_group *cxl_memdev_attribute_groups[] = {
&cxl_memdev_attribute_group,
&cxl_memdev_ram_attribute_group,
&cxl_memdev_pmem_attribute_group,
+ &cxl_memdev_security_attribute_group,
NULL,
};
@@ -397,17 +505,18 @@ EXPORT_SYMBOL_NS_GPL(is_cxl_memdev, CXL);
/**
* set_exclusive_cxl_commands() - atomically disable user cxl commands
- * @cxlds: The device state to operate on
+ * @mds: The device state to operate on
* @cmds: bitmap of commands to mark exclusive
*
* Grab the cxl_memdev_rwsem in write mode to flush in-flight
* invocations of the ioctl path and then disable future execution of
* commands with the command ids set in @cmds.
*/
-void set_exclusive_cxl_commands(struct cxl_dev_state *cxlds, unsigned long *cmds)
+void set_exclusive_cxl_commands(struct cxl_memdev_state *mds,
+ unsigned long *cmds)
{
down_write(&cxl_memdev_rwsem);
- bitmap_or(cxlds->exclusive_cmds, cxlds->exclusive_cmds, cmds,
+ bitmap_or(mds->exclusive_cmds, mds->exclusive_cmds, cmds,
CXL_MEM_COMMAND_ID_MAX);
up_write(&cxl_memdev_rwsem);
}
@@ -415,23 +524,34 @@ EXPORT_SYMBOL_NS_GPL(set_exclusive_cxl_commands, CXL);
/**
* clear_exclusive_cxl_commands() - atomically enable user cxl commands
- * @cxlds: The device state to modify
+ * @mds: The device state to modify
* @cmds: bitmap of commands to mark available for userspace
*/
-void clear_exclusive_cxl_commands(struct cxl_dev_state *cxlds, unsigned long *cmds)
+void clear_exclusive_cxl_commands(struct cxl_memdev_state *mds,
+ unsigned long *cmds)
{
down_write(&cxl_memdev_rwsem);
- bitmap_andnot(cxlds->exclusive_cmds, cxlds->exclusive_cmds, cmds,
+ bitmap_andnot(mds->exclusive_cmds, mds->exclusive_cmds, cmds,
CXL_MEM_COMMAND_ID_MAX);
up_write(&cxl_memdev_rwsem);
}
EXPORT_SYMBOL_NS_GPL(clear_exclusive_cxl_commands, CXL);
+static void cxl_memdev_security_shutdown(struct device *dev)
+{
+ struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
+ struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
+
+ if (mds->security.poll)
+ cancel_delayed_work_sync(&mds->security.poll_dwork);
+}
+
static void cxl_memdev_shutdown(struct device *dev)
{
struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
down_write(&cxl_memdev_rwsem);
+ cxl_memdev_security_shutdown(dev);
cxlmd->cxlds = NULL;
up_write(&cxl_memdev_rwsem);
}
@@ -511,10 +631,12 @@ static long cxl_memdev_ioctl(struct file *file, unsigned int cmd,
unsigned long arg)
{
struct cxl_memdev *cxlmd = file->private_data;
+ struct cxl_dev_state *cxlds;
int rc = -ENXIO;
down_read(&cxl_memdev_rwsem);
- if (cxlmd->cxlds)
+ cxlds = cxlmd->cxlds;
+ if (cxlds && cxlds->type == CXL_DEVTYPE_CLASSMEM)
rc = __cxl_memdev_ioctl(cxlmd, cmd, arg);
up_read(&cxl_memdev_rwsem);
@@ -542,6 +664,316 @@ static int cxl_memdev_release_file(struct inode *inode, struct file *file)
return 0;
}
+/**
+ * cxl_mem_get_fw_info - Get Firmware info
+ * @mds: The device data for the operation
+ *
+ * Retrieve firmware info for the device specified.
+ *
+ * Return: 0 if no error: or the result of the mailbox command.
+ *
+ * See CXL-3.0 8.2.9.3.1 Get FW Info
+ */
+static int cxl_mem_get_fw_info(struct cxl_memdev_state *mds)
+{
+ struct cxl_mbox_get_fw_info info;
+ struct cxl_mbox_cmd mbox_cmd;
+ int rc;
+
+ mbox_cmd = (struct cxl_mbox_cmd) {
+ .opcode = CXL_MBOX_OP_GET_FW_INFO,
+ .size_out = sizeof(info),
+ .payload_out = &info,
+ };
+
+ rc = cxl_internal_send_cmd(mds, &mbox_cmd);
+ if (rc < 0)
+ return rc;
+
+ mds->fw.num_slots = info.num_slots;
+ mds->fw.cur_slot = FIELD_GET(CXL_FW_INFO_SLOT_INFO_CUR_MASK,
+ info.slot_info);
+
+ return 0;
+}
+
+/**
+ * cxl_mem_activate_fw - Activate Firmware
+ * @mds: The device data for the operation
+ * @slot: slot number to activate
+ *
+ * Activate firmware in a given slot for the device specified.
+ *
+ * Return: 0 if no error: or the result of the mailbox command.
+ *
+ * See CXL-3.0 8.2.9.3.3 Activate FW
+ */
+static int cxl_mem_activate_fw(struct cxl_memdev_state *mds, int slot)
+{
+ struct cxl_mbox_activate_fw activate;
+ struct cxl_mbox_cmd mbox_cmd;
+
+ if (slot == 0 || slot > mds->fw.num_slots)
+ return -EINVAL;
+
+ mbox_cmd = (struct cxl_mbox_cmd) {
+ .opcode = CXL_MBOX_OP_ACTIVATE_FW,
+ .size_in = sizeof(activate),
+ .payload_in = &activate,
+ };
+
+ /* Only offline activation supported for now */
+ activate.action = CXL_FW_ACTIVATE_OFFLINE;
+ activate.slot = slot;
+
+ return cxl_internal_send_cmd(mds, &mbox_cmd);
+}
+
+/**
+ * cxl_mem_abort_fw_xfer - Abort an in-progress FW transfer
+ * @mds: The device data for the operation
+ *
+ * Abort an in-progress firmware transfer for the device specified.
+ *
+ * Return: 0 if no error: or the result of the mailbox command.
+ *
+ * See CXL-3.0 8.2.9.3.2 Transfer FW
+ */
+static int cxl_mem_abort_fw_xfer(struct cxl_memdev_state *mds)
+{
+ struct cxl_mbox_transfer_fw *transfer;
+ struct cxl_mbox_cmd mbox_cmd;
+ int rc;
+
+ transfer = kzalloc(struct_size(transfer, data, 0), GFP_KERNEL);
+ if (!transfer)
+ return -ENOMEM;
+
+ /* Set a 1s poll interval and a total wait time of 30s */
+ mbox_cmd = (struct cxl_mbox_cmd) {
+ .opcode = CXL_MBOX_OP_TRANSFER_FW,
+ .size_in = sizeof(*transfer),
+ .payload_in = transfer,
+ .poll_interval_ms = 1000,
+ .poll_count = 30,
+ };
+
+ transfer->action = CXL_FW_TRANSFER_ACTION_ABORT;
+
+ rc = cxl_internal_send_cmd(mds, &mbox_cmd);
+ kfree(transfer);
+ return rc;
+}
+
+static void cxl_fw_cleanup(struct fw_upload *fwl)
+{
+ struct cxl_memdev_state *mds = fwl->dd_handle;
+
+ mds->fw.next_slot = 0;
+}
+
+static int cxl_fw_do_cancel(struct fw_upload *fwl)
+{
+ struct cxl_memdev_state *mds = fwl->dd_handle;
+ struct cxl_dev_state *cxlds = &mds->cxlds;
+ struct cxl_memdev *cxlmd = cxlds->cxlmd;
+ int rc;
+
+ rc = cxl_mem_abort_fw_xfer(mds);
+ if (rc < 0)
+ dev_err(&cxlmd->dev, "Error aborting FW transfer: %d\n", rc);
+
+ return FW_UPLOAD_ERR_CANCELED;
+}
+
+static enum fw_upload_err cxl_fw_prepare(struct fw_upload *fwl, const u8 *data,
+ u32 size)
+{
+ struct cxl_memdev_state *mds = fwl->dd_handle;
+ struct cxl_mbox_transfer_fw *transfer;
+
+ if (!size)
+ return FW_UPLOAD_ERR_INVALID_SIZE;
+
+ mds->fw.oneshot = struct_size(transfer, data, size) <
+ mds->payload_size;
+
+ if (cxl_mem_get_fw_info(mds))
+ return FW_UPLOAD_ERR_HW_ERROR;
+
+ /*
+ * So far no state has been changed, hence no other cleanup is
+ * necessary. Simply return the cancelled status.
+ */
+ if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state))
+ return FW_UPLOAD_ERR_CANCELED;
+
+ return FW_UPLOAD_ERR_NONE;
+}
+
+static enum fw_upload_err cxl_fw_write(struct fw_upload *fwl, const u8 *data,
+ u32 offset, u32 size, u32 *written)
+{
+ struct cxl_memdev_state *mds = fwl->dd_handle;
+ struct cxl_dev_state *cxlds = &mds->cxlds;
+ struct cxl_memdev *cxlmd = cxlds->cxlmd;
+ struct cxl_mbox_transfer_fw *transfer;
+ struct cxl_mbox_cmd mbox_cmd;
+ u32 cur_size, remaining;
+ size_t size_in;
+ int rc;
+
+ *written = 0;
+
+ /* Offset has to be aligned to 128B (CXL-3.0 8.2.9.3.2 Table 8-57) */
+ if (!IS_ALIGNED(offset, CXL_FW_TRANSFER_ALIGNMENT)) {
+ dev_err(&cxlmd->dev,
+ "misaligned offset for FW transfer slice (%u)\n",
+ offset);
+ return FW_UPLOAD_ERR_RW_ERROR;
+ }
+
+ /*
+ * Pick transfer size based on mds->payload_size @size must bw 128-byte
+ * aligned, ->payload_size is a power of 2 starting at 256 bytes, and
+ * sizeof(*transfer) is 128. These constraints imply that @cur_size
+ * will always be 128b aligned.
+ */
+ cur_size = min_t(size_t, size, mds->payload_size - sizeof(*transfer));
+
+ remaining = size - cur_size;
+ size_in = struct_size(transfer, data, cur_size);
+
+ if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state))
+ return cxl_fw_do_cancel(fwl);
+
+ /*
+ * Slot numbers are 1-indexed
+ * cur_slot is the 0-indexed next_slot (i.e. 'cur_slot - 1 + 1')
+ * Check for rollover using modulo, and 1-index it by adding 1
+ */
+ mds->fw.next_slot = (mds->fw.cur_slot % mds->fw.num_slots) + 1;
+
+ /* Do the transfer via mailbox cmd */
+ transfer = kzalloc(size_in, GFP_KERNEL);
+ if (!transfer)
+ return FW_UPLOAD_ERR_RW_ERROR;
+
+ transfer->offset = cpu_to_le32(offset / CXL_FW_TRANSFER_ALIGNMENT);
+ memcpy(transfer->data, data + offset, cur_size);
+ if (mds->fw.oneshot) {
+ transfer->action = CXL_FW_TRANSFER_ACTION_FULL;
+ transfer->slot = mds->fw.next_slot;
+ } else {
+ if (offset == 0) {
+ transfer->action = CXL_FW_TRANSFER_ACTION_INITIATE;
+ } else if (remaining == 0) {
+ transfer->action = CXL_FW_TRANSFER_ACTION_END;
+ transfer->slot = mds->fw.next_slot;
+ } else {
+ transfer->action = CXL_FW_TRANSFER_ACTION_CONTINUE;
+ }
+ }
+
+ mbox_cmd = (struct cxl_mbox_cmd) {
+ .opcode = CXL_MBOX_OP_TRANSFER_FW,
+ .size_in = size_in,
+ .payload_in = transfer,
+ .poll_interval_ms = 1000,
+ .poll_count = 30,
+ };
+
+ rc = cxl_internal_send_cmd(mds, &mbox_cmd);
+ if (rc < 0) {
+ rc = FW_UPLOAD_ERR_RW_ERROR;
+ goto out_free;
+ }
+
+ *written = cur_size;
+
+ /* Activate FW if oneshot or if the last slice was written */
+ if (mds->fw.oneshot || remaining == 0) {
+ dev_dbg(&cxlmd->dev, "Activating firmware slot: %d\n",
+ mds->fw.next_slot);
+ rc = cxl_mem_activate_fw(mds, mds->fw.next_slot);
+ if (rc < 0) {
+ dev_err(&cxlmd->dev, "Error activating firmware: %d\n",
+ rc);
+ rc = FW_UPLOAD_ERR_HW_ERROR;
+ goto out_free;
+ }
+ }
+
+ rc = FW_UPLOAD_ERR_NONE;
+
+out_free:
+ kfree(transfer);
+ return rc;
+}
+
+static enum fw_upload_err cxl_fw_poll_complete(struct fw_upload *fwl)
+{
+ struct cxl_memdev_state *mds = fwl->dd_handle;
+
+ /*
+ * cxl_internal_send_cmd() handles background operations synchronously.
+ * No need to wait for completions here - any errors would've been
+ * reported and handled during the ->write() call(s).
+ * Just check if a cancel request was received, and return success.
+ */
+ if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state))
+ return cxl_fw_do_cancel(fwl);
+
+ return FW_UPLOAD_ERR_NONE;
+}
+
+static void cxl_fw_cancel(struct fw_upload *fwl)
+{
+ struct cxl_memdev_state *mds = fwl->dd_handle;
+
+ set_bit(CXL_FW_CANCEL, mds->fw.state);
+}
+
+static const struct fw_upload_ops cxl_memdev_fw_ops = {
+ .prepare = cxl_fw_prepare,
+ .write = cxl_fw_write,
+ .poll_complete = cxl_fw_poll_complete,
+ .cancel = cxl_fw_cancel,
+ .cleanup = cxl_fw_cleanup,
+};
+
+static void devm_cxl_remove_fw_upload(void *fwl)
+{
+ firmware_upload_unregister(fwl);
+}
+
+int cxl_memdev_setup_fw_upload(struct cxl_memdev_state *mds)
+{
+ struct cxl_dev_state *cxlds = &mds->cxlds;
+ struct device *dev = &cxlds->cxlmd->dev;
+ struct fw_upload *fwl;
+ int rc;
+
+ if (!test_bit(CXL_MEM_COMMAND_ID_GET_FW_INFO, mds->enabled_cmds))
+ return 0;
+
+ fwl = firmware_upload_register(THIS_MODULE, dev, dev_name(dev),
+ &cxl_memdev_fw_ops, mds);
+ if (IS_ERR(fwl))
+ return dev_err_probe(dev, PTR_ERR(fwl),
+ "Failed to register firmware loader\n");
+
+ rc = devm_add_action_or_reset(cxlds->dev, devm_cxl_remove_fw_upload,
+ fwl);
+ if (rc)
+ dev_err(dev,
+ "Failed to add firmware loader remove action: %d\n",
+ rc);
+
+ return rc;
+}
+EXPORT_SYMBOL_NS_GPL(cxl_memdev_setup_fw_upload, CXL);
+
static const struct file_operations cxl_memdev_fops = {
.owner = THIS_MODULE,
.unlocked_ioctl = cxl_memdev_ioctl,
@@ -551,6 +983,35 @@ static const struct file_operations cxl_memdev_fops = {
.llseek = noop_llseek,
};
+static void put_sanitize(void *data)
+{
+ struct cxl_memdev_state *mds = data;
+
+ sysfs_put(mds->security.sanitize_node);
+}
+
+static int cxl_memdev_security_init(struct cxl_memdev *cxlmd)
+{
+ struct cxl_dev_state *cxlds = cxlmd->cxlds;
+ struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
+ struct device *dev = &cxlmd->dev;
+ struct kernfs_node *sec;
+
+ sec = sysfs_get_dirent(dev->kobj.sd, "security");
+ if (!sec) {
+ dev_err(dev, "sysfs_get_dirent 'security' failed\n");
+ return -ENODEV;
+ }
+ mds->security.sanitize_node = sysfs_get_dirent(sec, "state");
+ sysfs_put(sec);
+ if (!mds->security.sanitize_node) {
+ dev_err(dev, "sysfs_get_dirent 'state' failed\n");
+ return -ENODEV;
+ }
+
+ return devm_add_action_or_reset(cxlds->dev, put_sanitize, mds);
+ }
+
struct cxl_memdev *devm_cxl_add_memdev(struct cxl_dev_state *cxlds)
{
struct cxl_memdev *cxlmd;
@@ -579,6 +1040,10 @@ struct cxl_memdev *devm_cxl_add_memdev(struct cxl_dev_state *cxlds)
if (rc)
goto err;
+ rc = cxl_memdev_security_init(cxlmd);
+ if (rc)
+ goto err;
+
rc = devm_add_action_or_reset(cxlds->dev, cxl_memdev_unregister, cxlmd);
if (rc)
return ERR_PTR(rc);
diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
index 67f4ab6..c7a7887 100644
--- a/drivers/cxl/core/pci.c
+++ b/drivers/cxl/core/pci.c
@@ -67,7 +67,7 @@ static int match_add_dports(struct pci_dev *pdev, void *data)
/**
* devm_cxl_port_enumerate_dports - enumerate downstream ports of the upstream port
- * @port: cxl_port whose ->uport is the upstream of dports to be enumerated
+ * @port: cxl_port whose ->uport_dev is the upstream of dports to be enumerated
*
* Returns a positive number of dports enumerated or a negative error
* code.
@@ -308,36 +308,17 @@ static void disable_hdm(void *_cxlhdm)
hdm + CXL_HDM_DECODER_CTRL_OFFSET);
}
-int devm_cxl_enable_hdm(struct cxl_port *port, struct cxl_hdm *cxlhdm)
+static int devm_cxl_enable_hdm(struct device *host, struct cxl_hdm *cxlhdm)
{
- void __iomem *hdm;
+ void __iomem *hdm = cxlhdm->regs.hdm_decoder;
u32 global_ctrl;
- /*
- * If the hdm capability was not mapped there is nothing to enable and
- * the caller is responsible for what happens next. For example,
- * emulate a passthrough decoder.
- */
- if (IS_ERR(cxlhdm))
- return 0;
-
- hdm = cxlhdm->regs.hdm_decoder;
global_ctrl = readl(hdm + CXL_HDM_DECODER_CTRL_OFFSET);
-
- /*
- * If the HDM decoder capability was enabled on entry, skip
- * registering disable_hdm() since this decode capability may be
- * owned by platform firmware.
- */
- if (global_ctrl & CXL_HDM_DECODER_ENABLE)
- return 0;
-
writel(global_ctrl | CXL_HDM_DECODER_ENABLE,
hdm + CXL_HDM_DECODER_CTRL_OFFSET);
- return devm_add_action_or_reset(&port->dev, disable_hdm, cxlhdm);
+ return devm_add_action_or_reset(host, disable_hdm, cxlhdm);
}
-EXPORT_SYMBOL_NS_GPL(devm_cxl_enable_hdm, CXL);
int cxl_dvsec_rr_decode(struct device *dev, int d,
struct cxl_endpoint_dvsec_info *info)
@@ -511,7 +492,7 @@ int cxl_hdm_decode_init(struct cxl_dev_state *cxlds, struct cxl_hdm *cxlhdm,
if (info->mem_enabled)
return 0;
- rc = devm_cxl_enable_hdm(port, cxlhdm);
+ rc = devm_cxl_enable_hdm(&port->dev, cxlhdm);
if (rc)
return rc;
@@ -622,7 +603,7 @@ static int cxl_cdat_read_table(struct device *dev,
*/
void read_cdat_data(struct cxl_port *port)
{
- struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport);
+ struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev);
struct device *host = cxlmd->dev.parent;
struct device *dev = &port->dev;
struct pci_doe_mb *cdat_doe;
diff --git a/drivers/cxl/core/pmem.c b/drivers/cxl/core/pmem.c
index f8c38d99..fc94f52 100644
--- a/drivers/cxl/core/pmem.c
+++ b/drivers/cxl/core/pmem.c
@@ -64,7 +64,7 @@ static int match_nvdimm_bridge(struct device *dev, void *data)
struct cxl_nvdimm_bridge *cxl_find_nvdimm_bridge(struct cxl_memdev *cxlmd)
{
- struct cxl_port *port = find_cxl_root(dev_get_drvdata(&cxlmd->dev));
+ struct cxl_port *port = find_cxl_root(cxlmd->endpoint);
struct device *dev;
if (!port)
diff --git a/drivers/cxl/core/pmu.c b/drivers/cxl/core/pmu.c
new file mode 100644
index 0000000..7684c843
--- /dev/null
+++ b/drivers/cxl/core/pmu.c
@@ -0,0 +1,68 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright(c) 2023 Huawei. All rights reserved. */
+
+#include <linux/device.h>
+#include <linux/slab.h>
+#include <linux/idr.h>
+#include <cxlmem.h>
+#include <pmu.h>
+#include <cxl.h>
+#include "core.h"
+
+static void cxl_pmu_release(struct device *dev)
+{
+ struct cxl_pmu *pmu = to_cxl_pmu(dev);
+
+ kfree(pmu);
+}
+
+const struct device_type cxl_pmu_type = {
+ .name = "cxl_pmu",
+ .release = cxl_pmu_release,
+};
+
+static void remove_dev(void *dev)
+{
+ device_del(dev);
+}
+
+int devm_cxl_pmu_add(struct device *parent, struct cxl_pmu_regs *regs,
+ int assoc_id, int index, enum cxl_pmu_type type)
+{
+ struct cxl_pmu *pmu;
+ struct device *dev;
+ int rc;
+
+ pmu = kzalloc(sizeof(*pmu), GFP_KERNEL);
+ if (!pmu)
+ return -ENOMEM;
+
+ pmu->assoc_id = assoc_id;
+ pmu->index = index;
+ pmu->type = type;
+ pmu->base = regs->pmu;
+ dev = &pmu->dev;
+ device_initialize(dev);
+ device_set_pm_not_required(dev);
+ dev->parent = parent;
+ dev->bus = &cxl_bus_type;
+ dev->type = &cxl_pmu_type;
+ switch (pmu->type) {
+ case CXL_PMU_MEMDEV:
+ rc = dev_set_name(dev, "pmu_mem%d.%d", assoc_id, index);
+ break;
+ }
+ if (rc)
+ goto err;
+
+ rc = device_add(dev);
+ if (rc)
+ goto err;
+
+ return devm_add_action_or_reset(parent, remove_dev, dev);
+
+err:
+ put_device(&pmu->dev);
+ return rc;
+}
+EXPORT_SYMBOL_NS_GPL(devm_cxl_pmu_add, CXL);
diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c
index e7c284c..724be84 100644
--- a/drivers/cxl/core/port.c
+++ b/drivers/cxl/core/port.c
@@ -56,6 +56,8 @@ static int cxl_device_id(const struct device *dev)
return CXL_DEVICE_MEMORY_EXPANDER;
if (dev->type == CXL_REGION_TYPE())
return CXL_DEVICE_REGION;
+ if (dev->type == &cxl_pmu_type)
+ return CXL_DEVICE_PMU;
return 0;
}
@@ -117,9 +119,9 @@ static ssize_t target_type_show(struct device *dev,
struct cxl_decoder *cxld = to_cxl_decoder(dev);
switch (cxld->target_type) {
- case CXL_DECODER_ACCELERATOR:
+ case CXL_DECODER_DEVMEM:
return sysfs_emit(buf, "accelerator\n");
- case CXL_DECODER_EXPANDER:
+ case CXL_DECODER_HOSTONLYMEM:
return sysfs_emit(buf, "expander\n");
}
return -ENXIO;
@@ -561,9 +563,9 @@ static void unregister_port(void *_port)
* unregistered while holding their parent port lock.
*/
if (!parent)
- lock_dev = port->uport;
+ lock_dev = port->uport_dev;
else if (is_cxl_root(parent))
- lock_dev = parent->uport;
+ lock_dev = parent->uport_dev;
else
lock_dev = &parent->dev;
@@ -583,7 +585,8 @@ static int devm_cxl_link_uport(struct device *host, struct cxl_port *port)
{
int rc;
- rc = sysfs_create_link(&port->dev.kobj, &port->uport->kobj, "uport");
+ rc = sysfs_create_link(&port->dev.kobj, &port->uport_dev->kobj,
+ "uport");
if (rc)
return rc;
return devm_add_action_or_reset(host, cxl_unlink_uport, port);
@@ -605,7 +608,7 @@ static int devm_cxl_link_parent_dport(struct device *host,
if (!parent_dport)
return 0;
- rc = sysfs_create_link(&port->dev.kobj, &parent_dport->dport->kobj,
+ rc = sysfs_create_link(&port->dev.kobj, &parent_dport->dport_dev->kobj,
"parent_dport");
if (rc)
return rc;
@@ -614,7 +617,7 @@ static int devm_cxl_link_parent_dport(struct device *host,
static struct lock_class_key cxl_port_key;
-static struct cxl_port *cxl_port_alloc(struct device *uport,
+static struct cxl_port *cxl_port_alloc(struct device *uport_dev,
resource_size_t component_reg_phys,
struct cxl_dport *parent_dport)
{
@@ -630,7 +633,7 @@ static struct cxl_port *cxl_port_alloc(struct device *uport,
if (rc < 0)
goto err;
port->id = rc;
- port->uport = uport;
+ port->uport_dev = uport_dev;
/*
* The top-level cxl_port "cxl_root" does not have a cxl_port as
@@ -658,12 +661,13 @@ static struct cxl_port *cxl_port_alloc(struct device *uport,
if (iter->host_bridge)
port->host_bridge = iter->host_bridge;
else if (parent_dport->rch)
- port->host_bridge = parent_dport->dport;
+ port->host_bridge = parent_dport->dport_dev;
else
- port->host_bridge = iter->uport;
- dev_dbg(uport, "host-bridge: %s\n", dev_name(port->host_bridge));
+ port->host_bridge = iter->uport_dev;
+ dev_dbg(uport_dev, "host-bridge: %s\n",
+ dev_name(port->host_bridge));
} else
- dev->parent = uport;
+ dev->parent = uport_dev;
port->component_reg_phys = component_reg_phys;
ida_init(&port->decoder_ida);
@@ -686,8 +690,38 @@ static struct cxl_port *cxl_port_alloc(struct device *uport,
return ERR_PTR(rc);
}
+static int cxl_setup_comp_regs(struct device *dev, struct cxl_register_map *map,
+ resource_size_t component_reg_phys)
+{
+ if (component_reg_phys == CXL_RESOURCE_NONE)
+ return 0;
+
+ *map = (struct cxl_register_map) {
+ .dev = dev,
+ .reg_type = CXL_REGLOC_RBI_COMPONENT,
+ .resource = component_reg_phys,
+ .max_size = CXL_COMPONENT_REG_BLOCK_SIZE,
+ };
+
+ return cxl_setup_regs(map);
+}
+
+static inline int cxl_port_setup_regs(struct cxl_port *port,
+ resource_size_t component_reg_phys)
+{
+ return cxl_setup_comp_regs(&port->dev, &port->comp_map,
+ component_reg_phys);
+}
+
+static inline int cxl_dport_setup_regs(struct cxl_dport *dport,
+ resource_size_t component_reg_phys)
+{
+ return cxl_setup_comp_regs(dport->dport_dev, &dport->comp_map,
+ component_reg_phys);
+}
+
static struct cxl_port *__devm_cxl_add_port(struct device *host,
- struct device *uport,
+ struct device *uport_dev,
resource_size_t component_reg_phys,
struct cxl_dport *parent_dport)
{
@@ -695,12 +729,12 @@ static struct cxl_port *__devm_cxl_add_port(struct device *host,
struct device *dev;
int rc;
- port = cxl_port_alloc(uport, component_reg_phys, parent_dport);
+ port = cxl_port_alloc(uport_dev, component_reg_phys, parent_dport);
if (IS_ERR(port))
return port;
dev = &port->dev;
- if (is_cxl_memdev(uport))
+ if (is_cxl_memdev(uport_dev))
rc = dev_set_name(dev, "endpoint%d", port->id);
else if (parent_dport)
rc = dev_set_name(dev, "port%d", port->id);
@@ -709,6 +743,10 @@ static struct cxl_port *__devm_cxl_add_port(struct device *host,
if (rc)
goto err;
+ rc = cxl_port_setup_regs(port, component_reg_phys);
+ if (rc)
+ goto err;
+
rc = device_add(dev);
if (rc)
goto err;
@@ -735,28 +773,29 @@ static struct cxl_port *__devm_cxl_add_port(struct device *host,
/**
* devm_cxl_add_port - register a cxl_port in CXL memory decode hierarchy
* @host: host device for devm operations
- * @uport: "physical" device implementing this upstream port
+ * @uport_dev: "physical" device implementing this upstream port
* @component_reg_phys: (optional) for configurable cxl_port instances
* @parent_dport: next hop up in the CXL memory decode hierarchy
*/
-struct cxl_port *devm_cxl_add_port(struct device *host, struct device *uport,
+struct cxl_port *devm_cxl_add_port(struct device *host,
+ struct device *uport_dev,
resource_size_t component_reg_phys,
struct cxl_dport *parent_dport)
{
struct cxl_port *port, *parent_port;
- port = __devm_cxl_add_port(host, uport, component_reg_phys,
+ port = __devm_cxl_add_port(host, uport_dev, component_reg_phys,
parent_dport);
parent_port = parent_dport ? parent_dport->port : NULL;
if (IS_ERR(port)) {
- dev_dbg(uport, "Failed to add%s%s%s: %ld\n",
+ dev_dbg(uport_dev, "Failed to add%s%s%s: %ld\n",
parent_port ? " port to " : "",
parent_port ? dev_name(&parent_port->dev) : "",
parent_port ? "" : " root port",
PTR_ERR(port));
} else {
- dev_dbg(uport, "%s added%s%s%s\n",
+ dev_dbg(uport_dev, "%s added%s%s%s\n",
dev_name(&port->dev),
parent_port ? " to " : "",
parent_port ? dev_name(&parent_port->dev) : "",
@@ -773,33 +812,34 @@ struct pci_bus *cxl_port_to_pci_bus(struct cxl_port *port)
if (is_cxl_root(port))
return NULL;
- if (dev_is_pci(port->uport)) {
- struct pci_dev *pdev = to_pci_dev(port->uport);
+ if (dev_is_pci(port->uport_dev)) {
+ struct pci_dev *pdev = to_pci_dev(port->uport_dev);
return pdev->subordinate;
}
- return xa_load(&cxl_root_buses, (unsigned long)port->uport);
+ return xa_load(&cxl_root_buses, (unsigned long)port->uport_dev);
}
EXPORT_SYMBOL_NS_GPL(cxl_port_to_pci_bus, CXL);
-static void unregister_pci_bus(void *uport)
+static void unregister_pci_bus(void *uport_dev)
{
- xa_erase(&cxl_root_buses, (unsigned long)uport);
+ xa_erase(&cxl_root_buses, (unsigned long)uport_dev);
}
-int devm_cxl_register_pci_bus(struct device *host, struct device *uport,
+int devm_cxl_register_pci_bus(struct device *host, struct device *uport_dev,
struct pci_bus *bus)
{
int rc;
- if (dev_is_pci(uport))
+ if (dev_is_pci(uport_dev))
return -EINVAL;
- rc = xa_insert(&cxl_root_buses, (unsigned long)uport, bus, GFP_KERNEL);
+ rc = xa_insert(&cxl_root_buses, (unsigned long)uport_dev, bus,
+ GFP_KERNEL);
if (rc)
return rc;
- return devm_add_action_or_reset(host, unregister_pci_bus, uport);
+ return devm_add_action_or_reset(host, unregister_pci_bus, uport_dev);
}
EXPORT_SYMBOL_NS_GPL(devm_cxl_register_pci_bus, CXL);
@@ -847,22 +887,22 @@ static struct cxl_dport *find_dport(struct cxl_port *port, int id)
return NULL;
}
-static int add_dport(struct cxl_port *port, struct cxl_dport *new)
+static int add_dport(struct cxl_port *port, struct cxl_dport *dport)
{
struct cxl_dport *dup;
int rc;
device_lock_assert(&port->dev);
- dup = find_dport(port, new->port_id);
+ dup = find_dport(port, dport->port_id);
if (dup) {
dev_err(&port->dev,
"unable to add dport%d-%s non-unique port id (%s)\n",
- new->port_id, dev_name(new->dport),
- dev_name(dup->dport));
+ dport->port_id, dev_name(dport->dport_dev),
+ dev_name(dup->dport_dev));
return -EBUSY;
}
- rc = xa_insert(&port->dports, (unsigned long)new->dport, new,
+ rc = xa_insert(&port->dports, (unsigned long)dport->dport_dev, dport,
GFP_KERNEL);
if (rc)
return rc;
@@ -895,8 +935,8 @@ static void cxl_dport_remove(void *data)
struct cxl_dport *dport = data;
struct cxl_port *port = dport->port;
- xa_erase(&port->dports, (unsigned long) dport->dport);
- put_device(dport->dport);
+ xa_erase(&port->dports, (unsigned long) dport->dport_dev);
+ put_device(dport->dport_dev);
}
static void cxl_dport_unlink(void *data)
@@ -920,7 +960,7 @@ __devm_cxl_add_dport(struct cxl_port *port, struct device *dport_dev,
int rc;
if (is_cxl_root(port))
- host = port->uport;
+ host = port->uport_dev;
else
host = &port->dev;
@@ -938,13 +978,29 @@ __devm_cxl_add_dport(struct cxl_port *port, struct device *dport_dev,
if (!dport)
return ERR_PTR(-ENOMEM);
- dport->dport = dport_dev;
- dport->port_id = port_id;
- dport->component_reg_phys = component_reg_phys;
- dport->port = port;
- if (rcrb != CXL_RESOURCE_NONE)
+ if (rcrb != CXL_RESOURCE_NONE) {
+ dport->rcrb.base = rcrb;
+ component_reg_phys = __rcrb_to_component(dport_dev, &dport->rcrb,
+ CXL_RCRB_DOWNSTREAM);
+ if (component_reg_phys == CXL_RESOURCE_NONE) {
+ dev_warn(dport_dev, "Invalid Component Registers in RCRB");
+ return ERR_PTR(-ENXIO);
+ }
+
dport->rch = true;
- dport->rcrb = rcrb;
+ }
+
+ if (component_reg_phys != CXL_RESOURCE_NONE)
+ dev_dbg(dport_dev, "Component Registers found for dport: %pa\n",
+ &component_reg_phys);
+
+ dport->dport_dev = dport_dev;
+ dport->port_id = port_id;
+ dport->port = port;
+
+ rc = cxl_dport_setup_regs(dport, component_reg_phys);
+ if (rc)
+ return ERR_PTR(rc);
cond_cxl_root_lock(port);
rc = add_dport(port, dport);
@@ -1004,14 +1060,12 @@ EXPORT_SYMBOL_NS_GPL(devm_cxl_add_dport, CXL);
* @port: the cxl_port that references this dport
* @dport_dev: firmware or PCI device representing the dport
* @port_id: identifier for this dport in a decoder's target list
- * @component_reg_phys: optional location of CXL component registers
* @rcrb: mandatory location of a Root Complex Register Block
*
* See CXL 3.0 9.11.8 CXL Devices Attached to an RCH
*/
struct cxl_dport *devm_cxl_add_rch_dport(struct cxl_port *port,
struct device *dport_dev, int port_id,
- resource_size_t component_reg_phys,
resource_size_t rcrb)
{
struct cxl_dport *dport;
@@ -1022,7 +1076,7 @@ struct cxl_dport *devm_cxl_add_rch_dport(struct cxl_port *port,
}
dport = __devm_cxl_add_dport(port, dport_dev, port_id,
- component_reg_phys, rcrb);
+ CXL_RESOURCE_NONE, rcrb);
if (IS_ERR(dport)) {
dev_dbg(dport_dev, "failed to add RCH dport to %s: %ld\n",
dev_name(&port->dev), PTR_ERR(dport));
@@ -1161,7 +1215,7 @@ static struct device *grandparent(struct device *dev)
static void delete_endpoint(void *data)
{
struct cxl_memdev *cxlmd = data;
- struct cxl_port *endpoint = dev_get_drvdata(&cxlmd->dev);
+ struct cxl_port *endpoint = cxlmd->endpoint;
struct cxl_port *parent_port;
struct device *parent;
@@ -1176,6 +1230,7 @@ static void delete_endpoint(void *data)
devm_release_action(parent, cxl_unlink_uport, endpoint);
devm_release_action(parent, unregister_port, endpoint);
}
+ cxlmd->endpoint = NULL;
device_unlock(parent);
put_device(parent);
out:
@@ -1187,7 +1242,7 @@ int cxl_endpoint_autoremove(struct cxl_memdev *cxlmd, struct cxl_port *endpoint)
struct device *dev = &cxlmd->dev;
get_device(&endpoint->dev);
- dev_set_drvdata(dev, endpoint);
+ cxlmd->endpoint = endpoint;
cxlmd->depth = endpoint->depth;
return devm_add_action_or_reset(dev, delete_endpoint, cxlmd);
}
@@ -1363,7 +1418,7 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd,
rc = PTR_ERR(port);
else {
dev_dbg(&cxlmd->dev, "add to new port %s:%s\n",
- dev_name(&port->dev), dev_name(port->uport));
+ dev_name(&port->dev), dev_name(port->uport_dev));
rc = cxl_add_ep(dport, &cxlmd->dev);
if (rc == -EBUSY) {
/*
@@ -1425,7 +1480,8 @@ int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd)
if (port) {
dev_dbg(&cxlmd->dev,
"found already registered port %s:%s\n",
- dev_name(&port->dev), dev_name(port->uport));
+ dev_name(&port->dev),
+ dev_name(port->uport_dev));
rc = cxl_add_ep(dport, &cxlmd->dev);
/*
@@ -1465,6 +1521,13 @@ int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd)
}
EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_ports, CXL);
+struct cxl_port *cxl_pci_find_port(struct pci_dev *pdev,
+ struct cxl_dport **dport)
+{
+ return find_cxl_port(pdev->dev.parent, dport);
+}
+EXPORT_SYMBOL_NS_GPL(cxl_pci_find_port, CXL);
+
struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd,
struct cxl_dport **dport)
{
@@ -1550,7 +1613,7 @@ static int cxl_decoder_init(struct cxl_port *port, struct cxl_decoder *cxld)
/* Pre initialize an "empty" decoder */
cxld->interleave_ways = 1;
cxld->interleave_granularity = PAGE_SIZE;
- cxld->target_type = CXL_DECODER_EXPANDER;
+ cxld->target_type = CXL_DECODER_HOSTONLYMEM;
cxld->hpa_range = (struct range) {
.start = 0,
.end = -1,
diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
index f822de4..e115ba3 100644
--- a/drivers/cxl/core/region.c
+++ b/drivers/cxl/core/region.c
@@ -125,10 +125,38 @@ static struct cxl_region_ref *cxl_rr_load(struct cxl_port *port,
return xa_load(&port->regions, (unsigned long)cxlr);
}
+static int cxl_region_invalidate_memregion(struct cxl_region *cxlr)
+{
+ if (!cpu_cache_has_invalidate_memregion()) {
+ if (IS_ENABLED(CONFIG_CXL_REGION_INVALIDATION_TEST)) {
+ dev_warn_once(
+ &cxlr->dev,
+ "Bypassing cpu_cache_invalidate_memregion() for testing!\n");
+ return 0;
+ } else {
+ dev_err(&cxlr->dev,
+ "Failed to synchronize CPU cache state\n");
+ return -ENXIO;
+ }
+ }
+
+ cpu_cache_invalidate_memregion(IORES_DESC_CXL);
+ return 0;
+}
+
static int cxl_region_decode_reset(struct cxl_region *cxlr, int count)
{
struct cxl_region_params *p = &cxlr->params;
- int i;
+ int i, rc = 0;
+
+ /*
+ * Before region teardown attempt to flush, and if the flush
+ * fails cancel the region teardown for data consistency
+ * concerns
+ */
+ rc = cxl_region_invalidate_memregion(cxlr);
+ if (rc)
+ return rc;
for (i = count - 1; i >= 0; i--) {
struct cxl_endpoint_decoder *cxled = p->targets[i];
@@ -136,7 +164,6 @@ static int cxl_region_decode_reset(struct cxl_region *cxlr, int count)
struct cxl_port *iter = cxled_to_port(cxled);
struct cxl_dev_state *cxlds = cxlmd->cxlds;
struct cxl_ep *ep;
- int rc = 0;
if (cxlds->rcd)
goto endpoint_reset;
@@ -155,14 +182,19 @@ static int cxl_region_decode_reset(struct cxl_region *cxlr, int count)
rc = cxld->reset(cxld);
if (rc)
return rc;
+ set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
}
endpoint_reset:
rc = cxled->cxld.reset(&cxled->cxld);
if (rc)
return rc;
+ set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
}
+ /* all decoders associated with this region have been torn down */
+ clear_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
+
return 0;
}
@@ -256,9 +288,19 @@ static ssize_t commit_store(struct device *dev, struct device_attribute *attr,
goto out;
}
- if (commit)
+ /*
+ * Invalidate caches before region setup to drop any speculative
+ * consumption of this address space
+ */
+ rc = cxl_region_invalidate_memregion(cxlr);
+ if (rc)
+ return rc;
+
+ if (commit) {
rc = cxl_region_decode_commit(cxlr);
- else {
+ if (rc == 0)
+ p->state = CXL_CONFIG_COMMIT;
+ } else {
p->state = CXL_CONFIG_RESET_PENDING;
up_write(&cxl_region_rwsem);
device_release_driver(&cxlr->dev);
@@ -268,18 +310,20 @@ static ssize_t commit_store(struct device *dev, struct device_attribute *attr,
* The lock was dropped, so need to revalidate that the reset is
* still pending.
*/
- if (p->state == CXL_CONFIG_RESET_PENDING)
+ if (p->state == CXL_CONFIG_RESET_PENDING) {
rc = cxl_region_decode_reset(cxlr, p->interleave_ways);
+ /*
+ * Revert to committed since there may still be active
+ * decoders associated with this region, or move forward
+ * to active to mark the reset successful
+ */
+ if (rc)
+ p->state = CXL_CONFIG_COMMIT;
+ else
+ p->state = CXL_CONFIG_ACTIVE;
+ }
}
- if (rc)
- goto out;
-
- if (commit)
- p->state = CXL_CONFIG_COMMIT;
- else if (p->state == CXL_CONFIG_RESET_PENDING)
- p->state = CXL_CONFIG_ACTIVE;
-
out:
up_write(&cxl_region_rwsem);
@@ -809,6 +853,18 @@ static int cxl_rr_alloc_decoder(struct cxl_port *port, struct cxl_region *cxlr,
return -EBUSY;
}
+ /*
+ * Endpoints should already match the region type, but backstop that
+ * assumption with an assertion. Switch-decoders change mapping-type
+ * based on what is mapped when they are assigned to a region.
+ */
+ dev_WARN_ONCE(&cxlr->dev,
+ port == cxled_to_port(cxled) &&
+ cxld->target_type != cxlr->type,
+ "%s:%s mismatch decoder type %d -> %d\n",
+ dev_name(&cxled_to_memdev(cxled)->dev),
+ dev_name(&cxld->dev), cxld->target_type, cxlr->type);
+ cxld->target_type = cxlr->type;
cxl_rr->decoder = cxld;
return 0;
}
@@ -906,10 +962,10 @@ static int cxl_port_attach_region(struct cxl_port *port,
dev_dbg(&cxlr->dev,
"%s:%s %s add: %s:%s @ %d next: %s nr_eps: %d nr_targets: %d\n",
- dev_name(port->uport), dev_name(&port->dev),
+ dev_name(port->uport_dev), dev_name(&port->dev),
dev_name(&cxld->dev), dev_name(&cxlmd->dev),
dev_name(&cxled->cxld.dev), pos,
- ep ? ep->next ? dev_name(ep->next->uport) :
+ ep ? ep->next ? dev_name(ep->next->uport_dev) :
dev_name(&cxlmd->dev) :
"none",
cxl_rr->nr_eps, cxl_rr->nr_targets);
@@ -984,7 +1040,7 @@ static int check_last_peer(struct cxl_endpoint_decoder *cxled,
*/
if (pos < distance) {
dev_dbg(&cxlr->dev, "%s:%s: cannot host %s:%s at %d\n",
- dev_name(port->uport), dev_name(&port->dev),
+ dev_name(port->uport_dev), dev_name(&port->dev),
dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
return -ENXIO;
}
@@ -994,7 +1050,7 @@ static int check_last_peer(struct cxl_endpoint_decoder *cxled,
if (ep->dport != ep_peer->dport) {
dev_dbg(&cxlr->dev,
"%s:%s: %s:%s pos %d mismatched peer %s:%s\n",
- dev_name(port->uport), dev_name(&port->dev),
+ dev_name(port->uport_dev), dev_name(&port->dev),
dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos,
dev_name(&cxlmd_peer->dev),
dev_name(&cxled_peer->cxld.dev));
@@ -1026,7 +1082,7 @@ static int cxl_port_setup_targets(struct cxl_port *port,
*/
if (!is_power_of_2(cxl_rr->nr_targets)) {
dev_dbg(&cxlr->dev, "%s:%s: invalid target count %d\n",
- dev_name(port->uport), dev_name(&port->dev),
+ dev_name(port->uport_dev), dev_name(&port->dev),
cxl_rr->nr_targets);
return -EINVAL;
}
@@ -1076,7 +1132,7 @@ static int cxl_port_setup_targets(struct cxl_port *port,
rc = granularity_to_eig(parent_ig, &peig);
if (rc) {
dev_dbg(&cxlr->dev, "%s:%s: invalid parent granularity: %d\n",
- dev_name(parent_port->uport),
+ dev_name(parent_port->uport_dev),
dev_name(&parent_port->dev), parent_ig);
return rc;
}
@@ -1084,7 +1140,7 @@ static int cxl_port_setup_targets(struct cxl_port *port,
rc = ways_to_eiw(parent_iw, &peiw);
if (rc) {
dev_dbg(&cxlr->dev, "%s:%s: invalid parent interleave: %d\n",
- dev_name(parent_port->uport),
+ dev_name(parent_port->uport_dev),
dev_name(&parent_port->dev), parent_iw);
return rc;
}
@@ -1093,7 +1149,7 @@ static int cxl_port_setup_targets(struct cxl_port *port,
rc = ways_to_eiw(iw, &eiw);
if (rc) {
dev_dbg(&cxlr->dev, "%s:%s: invalid port interleave: %d\n",
- dev_name(port->uport), dev_name(&port->dev), iw);
+ dev_name(port->uport_dev), dev_name(&port->dev), iw);
return rc;
}
@@ -1113,7 +1169,7 @@ static int cxl_port_setup_targets(struct cxl_port *port,
rc = eig_to_granularity(eig, &ig);
if (rc) {
dev_dbg(&cxlr->dev, "%s:%s: invalid interleave: %d\n",
- dev_name(port->uport), dev_name(&port->dev),
+ dev_name(port->uport_dev), dev_name(&port->dev),
256 << eig);
return rc;
}
@@ -1126,11 +1182,11 @@ static int cxl_port_setup_targets(struct cxl_port *port,
((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) {
dev_err(&cxlr->dev,
"%s:%s %s expected iw: %d ig: %d %pr\n",
- dev_name(port->uport), dev_name(&port->dev),
+ dev_name(port->uport_dev), dev_name(&port->dev),
__func__, iw, ig, p->res);
dev_err(&cxlr->dev,
"%s:%s %s got iw: %d ig: %d state: %s %#llx:%#llx\n",
- dev_name(port->uport), dev_name(&port->dev),
+ dev_name(port->uport_dev), dev_name(&port->dev),
__func__, cxld->interleave_ways,
cxld->interleave_granularity,
(cxld->flags & CXL_DECODER_F_ENABLE) ?
@@ -1147,22 +1203,22 @@ static int cxl_port_setup_targets(struct cxl_port *port,
.end = p->res->end,
};
}
- dev_dbg(&cxlr->dev, "%s:%s iw: %d ig: %d\n", dev_name(port->uport),
+ dev_dbg(&cxlr->dev, "%s:%s iw: %d ig: %d\n", dev_name(port->uport_dev),
dev_name(&port->dev), iw, ig);
add_target:
if (cxl_rr->nr_targets_set == cxl_rr->nr_targets) {
dev_dbg(&cxlr->dev,
"%s:%s: targets full trying to add %s:%s at %d\n",
- dev_name(port->uport), dev_name(&port->dev),
+ dev_name(port->uport_dev), dev_name(&port->dev),
dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
return -ENXIO;
}
if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
if (cxlsd->target[cxl_rr->nr_targets_set] != ep->dport) {
dev_dbg(&cxlr->dev, "%s:%s: %s expected %s at %d\n",
- dev_name(port->uport), dev_name(&port->dev),
+ dev_name(port->uport_dev), dev_name(&port->dev),
dev_name(&cxlsd->cxld.dev),
- dev_name(ep->dport->dport),
+ dev_name(ep->dport->dport_dev),
cxl_rr->nr_targets_set);
return -ENXIO;
}
@@ -1172,8 +1228,8 @@ static int cxl_port_setup_targets(struct cxl_port *port,
out_target_set:
cxl_rr->nr_targets_set += inc;
dev_dbg(&cxlr->dev, "%s:%s target[%d] = %s for %s:%s @ %d\n",
- dev_name(port->uport), dev_name(&port->dev),
- cxl_rr->nr_targets_set - 1, dev_name(ep->dport->dport),
+ dev_name(port->uport_dev), dev_name(&port->dev),
+ cxl_rr->nr_targets_set - 1, dev_name(ep->dport->dport_dev),
dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
return 0;
@@ -1492,7 +1548,7 @@ static int cmp_decode_pos(const void *a, const void *b)
if (!dev) {
struct range *range = &cxled_a->cxld.hpa_range;
- dev_err(port->uport,
+ dev_err(port->uport_dev,
"failed to find decoder that maps %#llx-%#llx\n",
range->start, range->end);
goto err;
@@ -1507,14 +1563,15 @@ static int cmp_decode_pos(const void *a, const void *b)
put_device(dev);
if (a_pos < 0 || b_pos < 0) {
- dev_err(port->uport,
+ dev_err(port->uport_dev,
"failed to find shared decoder for %s and %s\n",
dev_name(cxlmd_a->dev.parent),
dev_name(cxlmd_b->dev.parent));
goto err;
}
- dev_dbg(port->uport, "%s comes %s %s\n", dev_name(cxlmd_a->dev.parent),
+ dev_dbg(port->uport_dev, "%s comes %s %s\n",
+ dev_name(cxlmd_a->dev.parent),
a_pos - b_pos < 0 ? "before" : "after",
dev_name(cxlmd_b->dev.parent));
@@ -1674,7 +1731,6 @@ static int cxl_region_attach(struct cxl_region *cxlr,
if (rc)
goto err_decrement;
p->state = CXL_CONFIG_ACTIVE;
- set_bit(CXL_REGION_F_INCOHERENT, &cxlr->flags);
}
cxled->cxld.interleave_ways = p->interleave_ways;
@@ -2059,11 +2115,11 @@ static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd,
if (rc)
goto err;
- rc = devm_add_action_or_reset(port->uport, unregister_region, cxlr);
+ rc = devm_add_action_or_reset(port->uport_dev, unregister_region, cxlr);
if (rc)
return ERR_PTR(rc);
- dev_dbg(port->uport, "%s: created %s\n",
+ dev_dbg(port->uport_dev, "%s: created %s\n",
dev_name(&cxlrd->cxlsd.cxld.dev), dev_name(dev));
return cxlr;
@@ -2103,7 +2159,7 @@ static struct cxl_region *__create_region(struct cxl_root_decoder *cxlrd,
return ERR_PTR(-EBUSY);
}
- return devm_cxl_add_region(cxlrd, id, mode, CXL_DECODER_EXPANDER);
+ return devm_cxl_add_region(cxlrd, id, mode, CXL_DECODER_HOSTONLYMEM);
}
static ssize_t create_pmem_region_store(struct device *dev,
@@ -2191,7 +2247,7 @@ static ssize_t delete_region_store(struct device *dev,
if (IS_ERR(cxlr))
return PTR_ERR(cxlr);
- devm_release_action(port->uport, unregister_region, cxlr);
+ devm_release_action(port->uport_dev, unregister_region, cxlr);
put_device(&cxlr->dev);
return len;
@@ -2356,7 +2412,8 @@ int cxl_get_poison_by_endpoint(struct cxl_port *port)
rc = device_for_each_child(&port->dev, &ctx, poison_by_decoder);
if (rc == 1)
- rc = cxl_get_poison_unmapped(to_cxl_memdev(port->uport), &ctx);
+ rc = cxl_get_poison_unmapped(to_cxl_memdev(port->uport_dev),
+ &ctx);
up_read(&cxl_region_rwsem);
return rc;
@@ -2732,7 +2789,7 @@ static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
err:
up_write(&cxl_region_rwsem);
- devm_release_action(port->uport, unregister_region, cxlr);
+ devm_release_action(port->uport_dev, unregister_region, cxlr);
return ERR_PTR(rc);
}
@@ -2803,30 +2860,6 @@ int cxl_add_to_region(struct cxl_port *root, struct cxl_endpoint_decoder *cxled)
}
EXPORT_SYMBOL_NS_GPL(cxl_add_to_region, CXL);
-static int cxl_region_invalidate_memregion(struct cxl_region *cxlr)
-{
- if (!test_bit(CXL_REGION_F_INCOHERENT, &cxlr->flags))
- return 0;
-
- if (!cpu_cache_has_invalidate_memregion()) {
- if (IS_ENABLED(CONFIG_CXL_REGION_INVALIDATION_TEST)) {
- dev_warn_once(
- &cxlr->dev,
- "Bypassing cpu_cache_invalidate_memregion() for testing!\n");
- clear_bit(CXL_REGION_F_INCOHERENT, &cxlr->flags);
- return 0;
- } else {
- dev_err(&cxlr->dev,
- "Failed to synchronize CPU cache state\n");
- return -ENXIO;
- }
- }
-
- cpu_cache_invalidate_memregion(IORES_DESC_CXL);
- clear_bit(CXL_REGION_F_INCOHERENT, &cxlr->flags);
- return 0;
-}
-
static int is_system_ram(struct resource *res, void *arg)
{
struct cxl_region *cxlr = arg;
@@ -2854,7 +2887,12 @@ static int cxl_region_probe(struct device *dev)
goto out;
}
- rc = cxl_region_invalidate_memregion(cxlr);
+ if (test_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags)) {
+ dev_err(&cxlr->dev,
+ "failed to activate, re-commit region and retry\n");
+ rc = -ENXIO;
+ goto out;
+ }
/*
* From this point on any path that changes the region's state away from
diff --git a/drivers/cxl/core/regs.c b/drivers/cxl/core/regs.c
index 1476a02..6281127 100644
--- a/drivers/cxl/core/regs.c
+++ b/drivers/cxl/core/regs.c
@@ -6,6 +6,7 @@
#include <linux/pci.h>
#include <cxlmem.h>
#include <cxlpci.h>
+#include <pmu.h>
#include "core.h"
@@ -199,11 +200,13 @@ void __iomem *devm_cxl_iomap_block(struct device *dev, resource_size_t addr,
return ret_val;
}
-int cxl_map_component_regs(struct device *dev, struct cxl_component_regs *regs,
- struct cxl_register_map *map, unsigned long map_mask)
+int cxl_map_component_regs(const struct cxl_register_map *map,
+ struct cxl_component_regs *regs,
+ unsigned long map_mask)
{
+ struct device *dev = map->dev;
struct mapinfo {
- struct cxl_reg_map *rmap;
+ const struct cxl_reg_map *rmap;
void __iomem **addr;
} mapinfo[] = {
{ &map->component_map.hdm_decoder, ®s->hdm_decoder },
@@ -231,13 +234,13 @@ int cxl_map_component_regs(struct device *dev, struct cxl_component_regs *regs,
}
EXPORT_SYMBOL_NS_GPL(cxl_map_component_regs, CXL);
-int cxl_map_device_regs(struct device *dev,
- struct cxl_device_regs *regs,
- struct cxl_register_map *map)
+int cxl_map_device_regs(const struct cxl_register_map *map,
+ struct cxl_device_regs *regs)
{
+ struct device *dev = map->dev;
resource_size_t phys_addr = map->resource;
struct mapinfo {
- struct cxl_reg_map *rmap;
+ const struct cxl_reg_map *rmap;
void __iomem **addr;
} mapinfo[] = {
{ &map->device_map.status, ®s->status, },
@@ -286,23 +289,30 @@ static bool cxl_decode_regblock(struct pci_dev *pdev, u32 reg_lo, u32 reg_hi,
}
/**
- * cxl_find_regblock() - Locate register blocks by type
+ * cxl_find_regblock_instance() - Locate a register block by type / index
* @pdev: The CXL PCI device to enumerate.
* @type: Register Block Indicator id
* @map: Enumeration output, clobbered on error
+ * @index: Index into which particular instance of a regblock wanted in the
+ * order found in register locator DVSEC.
*
* Return: 0 if register block enumerated, negative error code otherwise
*
* A CXL DVSEC may point to one or more register blocks, search for them
- * by @type.
+ * by @type and @index.
*/
-int cxl_find_regblock(struct pci_dev *pdev, enum cxl_regloc_type type,
- struct cxl_register_map *map)
+int cxl_find_regblock_instance(struct pci_dev *pdev, enum cxl_regloc_type type,
+ struct cxl_register_map *map, int index)
{
u32 regloc_size, regblocks;
+ int instance = 0;
int regloc, i;
- map->resource = CXL_RESOURCE_NONE;
+ *map = (struct cxl_register_map) {
+ .dev = &pdev->dev,
+ .resource = CXL_RESOURCE_NONE,
+ };
+
regloc = pci_find_dvsec_capability(pdev, PCI_DVSEC_VENDOR_ID_CXL,
CXL_DVSEC_REG_LOCATOR);
if (!regloc)
@@ -323,20 +333,148 @@ int cxl_find_regblock(struct pci_dev *pdev, enum cxl_regloc_type type,
if (!cxl_decode_regblock(pdev, reg_lo, reg_hi, map))
continue;
- if (map->reg_type == type)
- return 0;
+ if (map->reg_type == type) {
+ if (index == instance)
+ return 0;
+ instance++;
+ }
}
map->resource = CXL_RESOURCE_NONE;
return -ENODEV;
}
+EXPORT_SYMBOL_NS_GPL(cxl_find_regblock_instance, CXL);
+
+/**
+ * cxl_find_regblock() - Locate register blocks by type
+ * @pdev: The CXL PCI device to enumerate.
+ * @type: Register Block Indicator id
+ * @map: Enumeration output, clobbered on error
+ *
+ * Return: 0 if register block enumerated, negative error code otherwise
+ *
+ * A CXL DVSEC may point to one or more register blocks, search for them
+ * by @type.
+ */
+int cxl_find_regblock(struct pci_dev *pdev, enum cxl_regloc_type type,
+ struct cxl_register_map *map)
+{
+ return cxl_find_regblock_instance(pdev, type, map, 0);
+}
EXPORT_SYMBOL_NS_GPL(cxl_find_regblock, CXL);
-resource_size_t cxl_rcrb_to_component(struct device *dev,
- resource_size_t rcrb,
- enum cxl_rcrb which)
+/**
+ * cxl_count_regblock() - Count instances of a given regblock type.
+ * @pdev: The CXL PCI device to enumerate.
+ * @type: Register Block Indicator id
+ *
+ * Some regblocks may be repeated. Count how many instances.
+ *
+ * Return: count of matching regblocks.
+ */
+int cxl_count_regblock(struct pci_dev *pdev, enum cxl_regloc_type type)
+{
+ struct cxl_register_map map;
+ int rc, count = 0;
+
+ while (1) {
+ rc = cxl_find_regblock_instance(pdev, type, &map, count);
+ if (rc)
+ return count;
+ count++;
+ }
+}
+EXPORT_SYMBOL_NS_GPL(cxl_count_regblock, CXL);
+
+int cxl_map_pmu_regs(struct pci_dev *pdev, struct cxl_pmu_regs *regs,
+ struct cxl_register_map *map)
+{
+ struct device *dev = &pdev->dev;
+ resource_size_t phys_addr;
+
+ phys_addr = map->resource;
+ regs->pmu = devm_cxl_iomap_block(dev, phys_addr, CXL_PMU_REGMAP_SIZE);
+ if (!regs->pmu)
+ return -ENOMEM;
+
+ return 0;
+}
+EXPORT_SYMBOL_NS_GPL(cxl_map_pmu_regs, CXL);
+
+static int cxl_map_regblock(struct cxl_register_map *map)
+{
+ struct device *dev = map->dev;
+
+ map->base = ioremap(map->resource, map->max_size);
+ if (!map->base) {
+ dev_err(dev, "failed to map registers\n");
+ return -ENOMEM;
+ }
+
+ dev_dbg(dev, "Mapped CXL Memory Device resource %pa\n", &map->resource);
+ return 0;
+}
+
+static void cxl_unmap_regblock(struct cxl_register_map *map)
+{
+ iounmap(map->base);
+ map->base = NULL;
+}
+
+static int cxl_probe_regs(struct cxl_register_map *map)
+{
+ struct cxl_component_reg_map *comp_map;
+ struct cxl_device_reg_map *dev_map;
+ struct device *dev = map->dev;
+ void __iomem *base = map->base;
+
+ switch (map->reg_type) {
+ case CXL_REGLOC_RBI_COMPONENT:
+ comp_map = &map->component_map;
+ cxl_probe_component_regs(dev, base, comp_map);
+ dev_dbg(dev, "Set up component registers\n");
+ break;
+ case CXL_REGLOC_RBI_MEMDEV:
+ dev_map = &map->device_map;
+ cxl_probe_device_regs(dev, base, dev_map);
+ if (!dev_map->status.valid || !dev_map->mbox.valid ||
+ !dev_map->memdev.valid) {
+ dev_err(dev, "registers not found: %s%s%s\n",
+ !dev_map->status.valid ? "status " : "",
+ !dev_map->mbox.valid ? "mbox " : "",
+ !dev_map->memdev.valid ? "memdev " : "");
+ return -ENXIO;
+ }
+
+ dev_dbg(dev, "Probing device registers...\n");
+ break;
+ default:
+ break;
+ }
+
+ return 0;
+}
+
+int cxl_setup_regs(struct cxl_register_map *map)
+{
+ int rc;
+
+ rc = cxl_map_regblock(map);
+ if (rc)
+ return rc;
+
+ rc = cxl_probe_regs(map);
+ cxl_unmap_regblock(map);
+
+ return rc;
+}
+EXPORT_SYMBOL_NS_GPL(cxl_setup_regs, CXL);
+
+resource_size_t __rcrb_to_component(struct device *dev, struct cxl_rcrb_info *ri,
+ enum cxl_rcrb which)
{
resource_size_t component_reg_phys;
+ resource_size_t rcrb = ri->base;
void __iomem *addr;
u32 bar0, bar1;
u16 cmd;
@@ -395,4 +533,12 @@ resource_size_t cxl_rcrb_to_component(struct device *dev,
return component_reg_phys;
}
-EXPORT_SYMBOL_NS_GPL(cxl_rcrb_to_component, CXL);
+
+resource_size_t cxl_rcd_component_reg_phys(struct device *dev,
+ struct cxl_dport *dport)
+{
+ if (!dport->rch)
+ return CXL_RESOURCE_NONE;
+ return __rcrb_to_component(dev, &dport->rcrb, CXL_RCRB_UPSTREAM);
+}
+EXPORT_SYMBOL_NS_GPL(cxl_rcd_component_reg_phys, CXL);
diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
index f93a2853..76d9256 100644
--- a/drivers/cxl/cxl.h
+++ b/drivers/cxl/cxl.h
@@ -56,7 +56,7 @@
#define CXL_HDM_DECODER0_CTRL_COMMIT BIT(9)
#define CXL_HDM_DECODER0_CTRL_COMMITTED BIT(10)
#define CXL_HDM_DECODER0_CTRL_COMMIT_ERROR BIT(11)
-#define CXL_HDM_DECODER0_CTRL_TYPE BIT(12)
+#define CXL_HDM_DECODER0_CTRL_HOSTONLY BIT(12)
#define CXL_HDM_DECODER0_TL_LOW(i) (0x20 * (i) + 0x24)
#define CXL_HDM_DECODER0_TL_HIGH(i) (0x20 * (i) + 0x28)
#define CXL_HDM_DECODER0_SKIP_LOW(i) CXL_HDM_DECODER0_TL_LOW(i)
@@ -176,14 +176,22 @@ static inline int ways_to_eiw(unsigned int ways, u8 *eiw)
/* CXL 2.0 8.2.8.4 Mailbox Registers */
#define CXLDEV_MBOX_CAPS_OFFSET 0x00
#define CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK GENMASK(4, 0)
+#define CXLDEV_MBOX_CAP_BG_CMD_IRQ BIT(6)
+#define CXLDEV_MBOX_CAP_IRQ_MSGNUM_MASK GENMASK(10, 7)
#define CXLDEV_MBOX_CTRL_OFFSET 0x04
#define CXLDEV_MBOX_CTRL_DOORBELL BIT(0)
+#define CXLDEV_MBOX_CTRL_BG_CMD_IRQ BIT(2)
#define CXLDEV_MBOX_CMD_OFFSET 0x08
#define CXLDEV_MBOX_CMD_COMMAND_OPCODE_MASK GENMASK_ULL(15, 0)
#define CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK GENMASK_ULL(36, 16)
#define CXLDEV_MBOX_STATUS_OFFSET 0x10
+#define CXLDEV_MBOX_STATUS_BG_CMD BIT(0)
#define CXLDEV_MBOX_STATUS_RET_CODE_MASK GENMASK_ULL(47, 32)
#define CXLDEV_MBOX_BG_CMD_STATUS_OFFSET 0x18
+#define CXLDEV_MBOX_BG_CMD_COMMAND_OPCODE_MASK GENMASK_ULL(15, 0)
+#define CXLDEV_MBOX_BG_CMD_COMMAND_PCT_MASK GENMASK_ULL(22, 16)
+#define CXLDEV_MBOX_BG_CMD_COMMAND_RC_MASK GENMASK_ULL(47, 32)
+#define CXLDEV_MBOX_BG_CMD_COMMAND_VENDOR_MASK GENMASK_ULL(63, 48)
#define CXLDEV_MBOX_PAYLOAD_OFFSET 0x20
/*
@@ -209,6 +217,10 @@ struct cxl_regs {
struct_group_tagged(cxl_device_regs, device_regs,
void __iomem *status, *mbox, *memdev;
);
+
+ struct_group_tagged(cxl_pmu_regs, pmu_regs,
+ void __iomem *pmu;
+ );
};
struct cxl_reg_map {
@@ -229,16 +241,23 @@ struct cxl_device_reg_map {
struct cxl_reg_map memdev;
};
+struct cxl_pmu_reg_map {
+ struct cxl_reg_map pmu;
+};
+
/**
* struct cxl_register_map - DVSEC harvested register block mapping parameters
+ * @dev: device for devm operations and logging
* @base: virtual base of the register-block-BAR + @block_offset
* @resource: physical resource base of the register block
* @max_size: maximum mapping size to perform register search
* @reg_type: see enum cxl_regloc_type
* @component_map: cxl_reg_map for component registers
* @device_map: cxl_reg_maps for device registers
+ * @pmu_map: cxl_reg_maps for CXL Performance Monitoring Units
*/
struct cxl_register_map {
+ struct device *dev;
void __iomem *base;
resource_size_t resource;
resource_size_t max_size;
@@ -246,6 +265,7 @@ struct cxl_register_map {
union {
struct cxl_component_reg_map component_map;
struct cxl_device_reg_map device_map;
+ struct cxl_pmu_reg_map pmu_map;
};
};
@@ -253,23 +273,24 @@ void cxl_probe_component_regs(struct device *dev, void __iomem *base,
struct cxl_component_reg_map *map);
void cxl_probe_device_regs(struct device *dev, void __iomem *base,
struct cxl_device_reg_map *map);
-int cxl_map_component_regs(struct device *dev, struct cxl_component_regs *regs,
- struct cxl_register_map *map,
+int cxl_map_component_regs(const struct cxl_register_map *map,
+ struct cxl_component_regs *regs,
unsigned long map_mask);
-int cxl_map_device_regs(struct device *dev, struct cxl_device_regs *regs,
- struct cxl_register_map *map);
+int cxl_map_device_regs(const struct cxl_register_map *map,
+ struct cxl_device_regs *regs);
+int cxl_map_pmu_regs(struct pci_dev *pdev, struct cxl_pmu_regs *regs,
+ struct cxl_register_map *map);
enum cxl_regloc_type;
+int cxl_count_regblock(struct pci_dev *pdev, enum cxl_regloc_type type);
+int cxl_find_regblock_instance(struct pci_dev *pdev, enum cxl_regloc_type type,
+ struct cxl_register_map *map, int index);
int cxl_find_regblock(struct pci_dev *pdev, enum cxl_regloc_type type,
struct cxl_register_map *map);
-
-enum cxl_rcrb {
- CXL_RCRB_DOWNSTREAM,
- CXL_RCRB_UPSTREAM,
-};
-resource_size_t cxl_rcrb_to_component(struct device *dev,
- resource_size_t rcrb,
- enum cxl_rcrb which);
+int cxl_setup_regs(struct cxl_register_map *map);
+struct cxl_dport;
+resource_size_t cxl_rcd_component_reg_phys(struct device *dev,
+ struct cxl_dport *dport);
#define CXL_RESOURCE_NONE ((resource_size_t) -1)
#define CXL_TARGET_STRLEN 20
@@ -290,8 +311,8 @@ resource_size_t cxl_rcrb_to_component(struct device *dev,
#define CXL_DECODER_F_MASK GENMASK(5, 0)
enum cxl_decoder_type {
- CXL_DECODER_ACCELERATOR = 2,
- CXL_DECODER_EXPANDER = 3,
+ CXL_DECODER_DEVMEM = 2,
+ CXL_DECODER_HOSTONLYMEM = 3,
};
/*
@@ -463,17 +484,19 @@ struct cxl_region_params {
};
/*
- * Flag whether this region needs to have its HPA span synchronized with
- * CPU cache state at region activation time.
- */
-#define CXL_REGION_F_INCOHERENT 0
-
-/*
* Indicate whether this region has been assembled by autodetection or
* userspace assembly. Prevent endpoint decoders outside of automatic
* detection from being added to the region.
*/
-#define CXL_REGION_F_AUTO 1
+#define CXL_REGION_F_AUTO 0
+
+/*
+ * Require that a committed region successfully complete a teardown once
+ * any of its associated decoders have been torn down. This maintains
+ * the commit state for the region since there are committed decoders,
+ * but blocks cxl_region_probe().
+ */
+#define CXL_REGION_F_NEEDS_RESET 1
/**
* struct cxl_region - CXL region
@@ -541,7 +564,7 @@ struct cxl_dax_region {
* downstream port devices to construct a CXL memory
* decode hierarchy.
* @dev: this port's device
- * @uport: PCI or platform device implementing the upstream port capability
+ * @uport_dev: PCI or platform device implementing the upstream port capability
* @host_bridge: Shortcut to the platform attach point for this port
* @id: id for port device-name
* @dports: cxl_dport instances referenced by decoders
@@ -549,6 +572,7 @@ struct cxl_dax_region {
* @regions: cxl_region_ref instances, regions mapped by this port
* @parent_dport: dport that points to this port in the parent
* @decoder_ida: allocator for decoder ids
+ * @comp_map: component register capability mappings
* @nr_dports: number of entries in @dports
* @hdm_end: track last allocated HDM decoder instance for allocation ordering
* @commit_end: cursor to track highest committed decoder for commit ordering
@@ -560,7 +584,7 @@ struct cxl_dax_region {
*/
struct cxl_port {
struct device dev;
- struct device *uport;
+ struct device *uport_dev;
struct device *host_bridge;
int id;
struct xarray dports;
@@ -568,6 +592,7 @@ struct cxl_port {
struct xarray regions;
struct cxl_dport *parent_dport;
struct ida decoder_ida;
+ struct cxl_register_map comp_map;
int nr_dports;
int hdm_end;
int commit_end;
@@ -587,20 +612,25 @@ cxl_find_dport_by_dev(struct cxl_port *port, const struct device *dport_dev)
return xa_load(&port->dports, (unsigned long)dport_dev);
}
+struct cxl_rcrb_info {
+ resource_size_t base;
+ u16 aer_cap;
+};
+
/**
* struct cxl_dport - CXL downstream port
- * @dport: PCI bridge or firmware device representing the downstream link
+ * @dport_dev: PCI bridge or firmware device representing the downstream link
+ * @comp_map: component register capability mappings
* @port_id: unique hardware identifier for dport in decoder target list
- * @component_reg_phys: downstream port component registers
- * @rcrb: base address for the Root Complex Register Block
+ * @rcrb: Data about the Root Complex Register Block layout
* @rch: Indicate whether this dport was enumerated in RCH or VH mode
* @port: reference to cxl_port that contains this downstream port
*/
struct cxl_dport {
- struct device *dport;
+ struct device *dport_dev;
+ struct cxl_register_map comp_map;
int port_id;
- resource_size_t component_reg_phys;
- resource_size_t rcrb;
+ struct cxl_rcrb_info rcrb;
bool rch;
struct cxl_port *port;
};
@@ -641,27 +671,30 @@ struct cxl_region_ref {
/*
* The platform firmware device hosting the root is also the top of the
* CXL port topology. All other CXL ports have another CXL port as their
- * parent and their ->uport / host device is out-of-line of the port
+ * parent and their ->uport_dev / host device is out-of-line of the port
* ancestry.
*/
static inline bool is_cxl_root(struct cxl_port *port)
{
- return port->uport == port->dev.parent;
+ return port->uport_dev == port->dev.parent;
}
bool is_cxl_port(const struct device *dev);
struct cxl_port *to_cxl_port(const struct device *dev);
struct pci_bus;
-int devm_cxl_register_pci_bus(struct device *host, struct device *uport,
+int devm_cxl_register_pci_bus(struct device *host, struct device *uport_dev,
struct pci_bus *bus);
struct pci_bus *cxl_port_to_pci_bus(struct cxl_port *port);
-struct cxl_port *devm_cxl_add_port(struct device *host, struct device *uport,
+struct cxl_port *devm_cxl_add_port(struct device *host,
+ struct device *uport_dev,
resource_size_t component_reg_phys,
struct cxl_dport *parent_dport);
struct cxl_port *find_cxl_root(struct cxl_port *port);
int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd);
void cxl_bus_rescan(void);
void cxl_bus_drain(void);
+struct cxl_port *cxl_pci_find_port(struct pci_dev *pdev,
+ struct cxl_dport **dport);
struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd,
struct cxl_dport **dport);
bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd);
@@ -671,7 +704,6 @@ struct cxl_dport *devm_cxl_add_dport(struct cxl_port *port,
resource_size_t component_reg_phys);
struct cxl_dport *devm_cxl_add_rch_dport(struct cxl_port *port,
struct device *dport_dev, int port_id,
- resource_size_t component_reg_phys,
resource_size_t rcrb);
struct cxl_decoder *to_cxl_decoder(struct device *dev);
@@ -710,7 +742,6 @@ struct cxl_endpoint_dvsec_info {
struct cxl_hdm;
struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port,
struct cxl_endpoint_dvsec_info *info);
-int devm_cxl_enable_hdm(struct cxl_port *port, struct cxl_hdm *cxlhdm);
int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm,
struct cxl_endpoint_dvsec_info *info);
int devm_cxl_add_passthrough_decoder(struct cxl_port *port);
@@ -750,6 +781,7 @@ void cxl_driver_unregister(struct cxl_driver *cxl_drv);
#define CXL_DEVICE_REGION 6
#define CXL_DEVICE_PMEM_REGION 7
#define CXL_DEVICE_DAX_REGION 8
+#define CXL_DEVICE_PMU 9
#define MODULE_ALIAS_CXL(type) MODULE_ALIAS("cxl:t" __stringify(type) "*")
#define CXL_MODALIAS_FMT "cxl:t%d"
diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h
index a2845a7..79e99c8 100644
--- a/drivers/cxl/cxlmem.h
+++ b/drivers/cxl/cxlmem.h
@@ -5,6 +5,7 @@
#include <uapi/linux/cxl_mem.h>
#include <linux/cdev.h>
#include <linux/uuid.h>
+#include <linux/rcuwait.h>
#include "cxl.h"
/* CXL 2.0 8.2.8.5.1.1 Memory Device Status Register */
@@ -38,6 +39,7 @@
* @detach_work: active memdev lost a port in its ancestry
* @cxl_nvb: coordinate removal of @cxl_nvd if present
* @cxl_nvd: optional bridge to an nvdimm if the device supports pmem
+ * @endpoint: connection to the CXL port topology for this memory device
* @id: id number of this memdev instance.
* @depth: endpoint port depth
*/
@@ -48,6 +50,7 @@ struct cxl_memdev {
struct work_struct detach_work;
struct cxl_nvdimm_bridge *cxl_nvb;
struct cxl_nvdimm *cxl_nvd;
+ struct cxl_port *endpoint;
int id;
int depth;
};
@@ -72,16 +75,18 @@ cxled_to_memdev(struct cxl_endpoint_decoder *cxled)
{
struct cxl_port *port = to_cxl_port(cxled->cxld.dev.parent);
- return to_cxl_memdev(port->uport);
+ return to_cxl_memdev(port->uport_dev);
}
bool is_cxl_memdev(const struct device *dev);
static inline bool is_cxl_endpoint(struct cxl_port *port)
{
- return is_cxl_memdev(port->uport);
+ return is_cxl_memdev(port->uport_dev);
}
struct cxl_memdev *devm_cxl_add_memdev(struct cxl_dev_state *cxlds);
+struct cxl_memdev_state;
+int cxl_memdev_setup_fw_upload(struct cxl_memdev_state *mds);
int devm_cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
resource_size_t base, resource_size_t len,
resource_size_t skipped);
@@ -108,6 +113,9 @@ static inline struct cxl_ep *cxl_ep_load(struct cxl_port *port,
* variable sized output commands, it tells the exact number of bytes
* written.
* @min_out: (input) internal command output payload size validation
+ * @poll_count: (input) Number of timeouts to attempt.
+ * @poll_interval_ms: (input) Time between mailbox background command polling
+ * interval timeouts.
* @return_code: (output) Error code returned from hardware.
*
* This is the primary mechanism used to send commands to the hardware.
@@ -123,6 +131,8 @@ struct cxl_mbox_cmd {
size_t size_in;
size_t size_out;
size_t min_out;
+ int poll_count;
+ int poll_interval_ms;
u16 return_code;
};
@@ -195,7 +205,7 @@ static inline int cxl_mbox_cmd_rc2errno(struct cxl_mbox_cmd *mbox_cmd)
*/
#define CXL_CAPACITY_MULTIPLIER SZ_256M
-/**
+/*
* Event Interrupt Policy
*
* CXL rev 3.0 section 8.2.9.2.4; Table 8-52
@@ -215,8 +225,8 @@ struct cxl_event_interrupt_policy {
/**
* struct cxl_event_state - Event log driver state
*
- * @event_buf: Buffer to receive event data
- * @event_log_lock: Serialize event_buf and log use
+ * @buf: Buffer to receive event data
+ * @log_lock: Serialize event_buf and log use
*/
struct cxl_event_state {
struct cxl_get_event_payload *buf;
@@ -254,6 +264,115 @@ struct cxl_poison_state {
struct mutex lock; /* Protect reads of poison list */
};
+/*
+ * Get FW Info
+ * CXL rev 3.0 section 8.2.9.3.1; Table 8-56
+ */
+struct cxl_mbox_get_fw_info {
+ u8 num_slots;
+ u8 slot_info;
+ u8 activation_cap;
+ u8 reserved[13];
+ char slot_1_revision[16];
+ char slot_2_revision[16];
+ char slot_3_revision[16];
+ char slot_4_revision[16];
+} __packed;
+
+#define CXL_FW_INFO_SLOT_INFO_CUR_MASK GENMASK(2, 0)
+#define CXL_FW_INFO_SLOT_INFO_NEXT_MASK GENMASK(5, 3)
+#define CXL_FW_INFO_SLOT_INFO_NEXT_SHIFT 3
+#define CXL_FW_INFO_ACTIVATION_CAP_HAS_LIVE_ACTIVATE BIT(0)
+
+/*
+ * Transfer FW Input Payload
+ * CXL rev 3.0 section 8.2.9.3.2; Table 8-57
+ */
+struct cxl_mbox_transfer_fw {
+ u8 action;
+ u8 slot;
+ u8 reserved[2];
+ __le32 offset;
+ u8 reserved2[0x78];
+ u8 data[];
+} __packed;
+
+#define CXL_FW_TRANSFER_ACTION_FULL 0x0
+#define CXL_FW_TRANSFER_ACTION_INITIATE 0x1
+#define CXL_FW_TRANSFER_ACTION_CONTINUE 0x2
+#define CXL_FW_TRANSFER_ACTION_END 0x3
+#define CXL_FW_TRANSFER_ACTION_ABORT 0x4
+
+/*
+ * CXL rev 3.0 section 8.2.9.3.2 mandates 128-byte alignment for FW packages
+ * and for each part transferred in a Transfer FW command.
+ */
+#define CXL_FW_TRANSFER_ALIGNMENT 128
+
+/*
+ * Activate FW Input Payload
+ * CXL rev 3.0 section 8.2.9.3.3; Table 8-58
+ */
+struct cxl_mbox_activate_fw {
+ u8 action;
+ u8 slot;
+} __packed;
+
+#define CXL_FW_ACTIVATE_ONLINE 0x0
+#define CXL_FW_ACTIVATE_OFFLINE 0x1
+
+/* FW state bits */
+#define CXL_FW_STATE_BITS 32
+#define CXL_FW_CANCEL BIT(0)
+
+/**
+ * struct cxl_fw_state - Firmware upload / activation state
+ *
+ * @state: fw_uploader state bitmask
+ * @oneshot: whether the fw upload fits in a single transfer
+ * @num_slots: Number of FW slots available
+ * @cur_slot: Slot number currently active
+ * @next_slot: Slot number for the new firmware
+ */
+struct cxl_fw_state {
+ DECLARE_BITMAP(state, CXL_FW_STATE_BITS);
+ bool oneshot;
+ int num_slots;
+ int cur_slot;
+ int next_slot;
+};
+
+/**
+ * struct cxl_security_state - Device security state
+ *
+ * @state: state of last security operation
+ * @poll: polling for sanitization is enabled, device has no mbox irq support
+ * @poll_tmo_secs: polling timeout
+ * @poll_dwork: polling work item
+ * @sanitize_node: sanitation sysfs file to notify
+ */
+struct cxl_security_state {
+ unsigned long state;
+ bool poll;
+ int poll_tmo_secs;
+ struct delayed_work poll_dwork;
+ struct kernfs_node *sanitize_node;
+};
+
+/*
+ * enum cxl_devtype - delineate type-2 from a generic type-3 device
+ * @CXL_DEVTYPE_DEVMEM - Vendor specific CXL Type-2 device implementing HDM-D or
+ * HDM-DB, no requirement that this device implements a
+ * mailbox, or other memory-device-standard manageability
+ * flows.
+ * @CXL_DEVTYPE_CLASSMEM - Common class definition of a CXL Type-3 device with
+ * HDM-H and class-mandatory memory device registers
+ */
+enum cxl_devtype {
+ CXL_DEVTYPE_DEVMEM,
+ CXL_DEVTYPE_CLASSMEM,
+};
+
/**
* struct cxl_dev_state - The driver device state
*
@@ -267,6 +386,36 @@ struct cxl_poison_state {
* @cxl_dvsec: Offset to the PCIe device DVSEC
* @rcd: operating in RCD mode (CXL 3.0 9.11.8 CXL Devices Attached to an RCH)
* @media_ready: Indicate whether the device media is usable
+ * @dpa_res: Overall DPA resource tree for the device
+ * @pmem_res: Active Persistent memory capacity configuration
+ * @ram_res: Active Volatile memory capacity configuration
+ * @component_reg_phys: register base of component registers
+ * @serial: PCIe Device Serial Number
+ * @type: Generic Memory Class device or Vendor Specific Memory device
+ */
+struct cxl_dev_state {
+ struct device *dev;
+ struct cxl_memdev *cxlmd;
+ struct cxl_regs regs;
+ int cxl_dvsec;
+ bool rcd;
+ bool media_ready;
+ struct resource dpa_res;
+ struct resource pmem_res;
+ struct resource ram_res;
+ resource_size_t component_reg_phys;
+ u64 serial;
+ enum cxl_devtype type;
+};
+
+/**
+ * struct cxl_memdev_state - Generic Type-3 Memory Device Class driver data
+ *
+ * CXL 8.1.12.1 PCI Header - Class Code Register Memory Device defines
+ * common memory device functionality like the presence of a mailbox and
+ * the functionality related to that like Identify Memory Device and Get
+ * Partition Info
+ * @cxlds: Core driver state common across Type-2 and Type-3 devices
* @payload_size: Size of space for payload
* (CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register)
* @lsa_size: Size of Label Storage Area
@@ -275,9 +424,6 @@ struct cxl_poison_state {
* @firmware_version: Firmware version for the memory device.
* @enabled_cmds: Hardware commands found enabled in CEL.
* @exclusive_cmds: Commands that are kernel-internal only
- * @dpa_res: Overall DPA resource tree for the device
- * @pmem_res: Active Persistent memory capacity configuration
- * @ram_res: Active Volatile memory capacity configuration
* @total_bytes: sum of all possible capacities
* @volatile_only_bytes: hard volatile capacity
* @persistent_only_bytes: hard persistent capacity
@@ -286,54 +432,48 @@ struct cxl_poison_state {
* @active_persistent_bytes: sum of hard + soft persistent
* @next_volatile_bytes: volatile capacity change pending device reset
* @next_persistent_bytes: persistent capacity change pending device reset
- * @component_reg_phys: register base of component registers
- * @info: Cached DVSEC information about the device.
- * @serial: PCIe Device Serial Number
* @event: event log driver state
* @poison: poison driver state info
+ * @fw: firmware upload / activation state
* @mbox_send: @dev specific transport for transmitting mailbox commands
*
- * See section 8.2.9.5.2 Capacity Configuration and Label Storage for
+ * See CXL 3.0 8.2.9.8.2 Capacity Configuration and Label Storage for
* details on capacity parameters.
*/
-struct cxl_dev_state {
- struct device *dev;
- struct cxl_memdev *cxlmd;
-
- struct cxl_regs regs;
- int cxl_dvsec;
-
- bool rcd;
- bool media_ready;
+struct cxl_memdev_state {
+ struct cxl_dev_state cxlds;
size_t payload_size;
size_t lsa_size;
struct mutex mbox_mutex; /* Protects device mailbox and firmware */
char firmware_version[0x10];
DECLARE_BITMAP(enabled_cmds, CXL_MEM_COMMAND_ID_MAX);
DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX);
-
- struct resource dpa_res;
- struct resource pmem_res;
- struct resource ram_res;
u64 total_bytes;
u64 volatile_only_bytes;
u64 persistent_only_bytes;
u64 partition_align_bytes;
-
u64 active_volatile_bytes;
u64 active_persistent_bytes;
u64 next_volatile_bytes;
u64 next_persistent_bytes;
-
- resource_size_t component_reg_phys;
- u64 serial;
-
struct cxl_event_state event;
struct cxl_poison_state poison;
+ struct cxl_security_state security;
+ struct cxl_fw_state fw;
- int (*mbox_send)(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd *cmd);
+ struct rcuwait mbox_wait;
+ int (*mbox_send)(struct cxl_memdev_state *mds,
+ struct cxl_mbox_cmd *cmd);
};
+static inline struct cxl_memdev_state *
+to_cxl_memdev_state(struct cxl_dev_state *cxlds)
+{
+ if (cxlds->type != CXL_DEVTYPE_CLASSMEM)
+ return NULL;
+ return container_of(cxlds, struct cxl_memdev_state, cxlds);
+}
+
enum cxl_opcode {
CXL_MBOX_OP_INVALID = 0x0000,
CXL_MBOX_OP_RAW = CXL_MBOX_OP_INVALID,
@@ -342,6 +482,7 @@ enum cxl_opcode {
CXL_MBOX_OP_GET_EVT_INT_POLICY = 0x0102,
CXL_MBOX_OP_SET_EVT_INT_POLICY = 0x0103,
CXL_MBOX_OP_GET_FW_INFO = 0x0200,
+ CXL_MBOX_OP_TRANSFER_FW = 0x0201,
CXL_MBOX_OP_ACTIVATE_FW = 0x0202,
CXL_MBOX_OP_SET_TIMESTAMP = 0x0301,
CXL_MBOX_OP_GET_SUPPORTED_LOGS = 0x0400,
@@ -362,6 +503,8 @@ enum cxl_opcode {
CXL_MBOX_OP_GET_SCAN_MEDIA_CAPS = 0x4303,
CXL_MBOX_OP_SCAN_MEDIA = 0x4304,
CXL_MBOX_OP_GET_SCAN_MEDIA = 0x4305,
+ CXL_MBOX_OP_SANITIZE = 0x4400,
+ CXL_MBOX_OP_SECURE_ERASE = 0x4401,
CXL_MBOX_OP_GET_SECURITY_STATE = 0x4500,
CXL_MBOX_OP_SET_PASSPHRASE = 0x4501,
CXL_MBOX_OP_DISABLE_PASSPHRASE = 0x4502,
@@ -692,18 +835,20 @@ enum {
CXL_PMEM_SEC_PASS_USER,
};
-int cxl_internal_send_cmd(struct cxl_dev_state *cxlds,
+int cxl_internal_send_cmd(struct cxl_memdev_state *mds,
struct cxl_mbox_cmd *cmd);
-int cxl_dev_state_identify(struct cxl_dev_state *cxlds);
+int cxl_dev_state_identify(struct cxl_memdev_state *mds);
int cxl_await_media_ready(struct cxl_dev_state *cxlds);
-int cxl_enumerate_cmds(struct cxl_dev_state *cxlds);
-int cxl_mem_create_range_info(struct cxl_dev_state *cxlds);
-struct cxl_dev_state *cxl_dev_state_create(struct device *dev);
-void set_exclusive_cxl_commands(struct cxl_dev_state *cxlds, unsigned long *cmds);
-void clear_exclusive_cxl_commands(struct cxl_dev_state *cxlds, unsigned long *cmds);
-void cxl_mem_get_event_records(struct cxl_dev_state *cxlds, u32 status);
-int cxl_set_timestamp(struct cxl_dev_state *cxlds);
-int cxl_poison_state_init(struct cxl_dev_state *cxlds);
+int cxl_enumerate_cmds(struct cxl_memdev_state *mds);
+int cxl_mem_create_range_info(struct cxl_memdev_state *mds);
+struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev);
+void set_exclusive_cxl_commands(struct cxl_memdev_state *mds,
+ unsigned long *cmds);
+void clear_exclusive_cxl_commands(struct cxl_memdev_state *mds,
+ unsigned long *cmds);
+void cxl_mem_get_event_records(struct cxl_memdev_state *mds, u32 status);
+int cxl_set_timestamp(struct cxl_memdev_state *mds);
+int cxl_poison_state_init(struct cxl_memdev_state *mds);
int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len,
struct cxl_region *cxlr);
int cxl_trigger_poison_list(struct cxl_memdev *cxlmd);
@@ -722,6 +867,8 @@ static inline void cxl_mem_active_dec(void)
}
#endif
+int cxl_mem_sanitize(struct cxl_memdev_state *mds, u16 cmd);
+
struct cxl_hdm {
struct cxl_component_regs regs;
unsigned int decoder_count;
diff --git a/drivers/cxl/cxlpci.h b/drivers/cxl/cxlpci.h
index 7c02e55..0fa4799 100644
--- a/drivers/cxl/cxlpci.h
+++ b/drivers/cxl/cxlpci.h
@@ -67,6 +67,7 @@ enum cxl_regloc_type {
CXL_REGLOC_RBI_COMPONENT,
CXL_REGLOC_RBI_VIRT,
CXL_REGLOC_RBI_MEMDEV,
+ CXL_REGLOC_RBI_PMU,
CXL_REGLOC_RBI_TYPES
};
diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c
index 519edd0..317c754 100644
--- a/drivers/cxl/mem.c
+++ b/drivers/cxl/mem.c
@@ -51,7 +51,6 @@ static int devm_cxl_add_endpoint(struct device *host, struct cxl_memdev *cxlmd,
struct cxl_port *parent_port = parent_dport->port;
struct cxl_dev_state *cxlds = cxlmd->cxlds;
struct cxl_port *endpoint, *iter, *down;
- resource_size_t component_reg_phys;
int rc;
/*
@@ -66,17 +65,8 @@ static int devm_cxl_add_endpoint(struct device *host, struct cxl_memdev *cxlmd,
ep->next = down;
}
- /*
- * The component registers for an RCD might come from the
- * host-bridge RCRB if they are not already mapped via the
- * typical register locator mechanism.
- */
- if (parent_dport->rch && cxlds->component_reg_phys == CXL_RESOURCE_NONE)
- component_reg_phys = cxl_rcrb_to_component(
- &cxlmd->dev, parent_dport->rcrb, CXL_RCRB_UPSTREAM);
- else
- component_reg_phys = cxlds->component_reg_phys;
- endpoint = devm_cxl_add_port(host, &cxlmd->dev, component_reg_phys,
+ endpoint = devm_cxl_add_port(host, &cxlmd->dev,
+ cxlds->component_reg_phys,
parent_dport);
if (IS_ERR(endpoint))
return PTR_ERR(endpoint);
@@ -117,6 +107,7 @@ DEFINE_DEBUGFS_ATTRIBUTE(cxl_poison_clear_fops, NULL,
static int cxl_mem_probe(struct device *dev)
{
struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
+ struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
struct cxl_dev_state *cxlds = cxlmd->cxlds;
struct device *endpoint_parent;
struct cxl_port *parent_port;
@@ -141,10 +132,10 @@ static int cxl_mem_probe(struct device *dev)
dentry = cxl_debugfs_create_dir(dev_name(dev));
debugfs_create_devm_seqfile(dev, "dpamem", dentry, cxl_mem_dpa_show);
- if (test_bit(CXL_POISON_ENABLED_INJECT, cxlds->poison.enabled_cmds))
+ if (test_bit(CXL_POISON_ENABLED_INJECT, mds->poison.enabled_cmds))
debugfs_create_file("inject_poison", 0200, dentry, cxlmd,
&cxl_poison_inject_fops);
- if (test_bit(CXL_POISON_ENABLED_CLEAR, cxlds->poison.enabled_cmds))
+ if (test_bit(CXL_POISON_ENABLED_CLEAR, mds->poison.enabled_cmds))
debugfs_create_file("clear_poison", 0200, dentry, cxlmd,
&cxl_poison_clear_fops);
@@ -163,7 +154,7 @@ static int cxl_mem_probe(struct device *dev)
}
if (dport->rch)
- endpoint_parent = parent_port->uport;
+ endpoint_parent = parent_port->uport_dev;
else
endpoint_parent = &parent_port->dev;
@@ -227,9 +218,12 @@ static umode_t cxl_mem_visible(struct kobject *kobj, struct attribute *a, int n)
{
if (a == &dev_attr_trigger_poison_list.attr) {
struct device *dev = kobj_to_dev(kobj);
+ struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
+ struct cxl_memdev_state *mds =
+ to_cxl_memdev_state(cxlmd->cxlds);
if (!test_bit(CXL_POISON_ENABLED_LIST,
- to_cxl_memdev(dev)->cxlds->poison.enabled_cmds))
+ mds->poison.enabled_cmds))
return 0;
}
return a->mode;
diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
index 0872f22..1cb1494 100644
--- a/drivers/cxl/pci.c
+++ b/drivers/cxl/pci.c
@@ -13,6 +13,7 @@
#include "cxlmem.h"
#include "cxlpci.h"
#include "cxl.h"
+#include "pmu.h"
/**
* DOC: cxl pci
@@ -84,9 +85,92 @@ static int cxl_pci_mbox_wait_for_doorbell(struct cxl_dev_state *cxlds)
status & CXLMDEV_DEV_FATAL ? " fatal" : "", \
status & CXLMDEV_FW_HALT ? " firmware-halt" : "")
+struct cxl_dev_id {
+ struct cxl_dev_state *cxlds;
+};
+
+static int cxl_request_irq(struct cxl_dev_state *cxlds, int irq,
+ irq_handler_t handler, irq_handler_t thread_fn)
+{
+ struct device *dev = cxlds->dev;
+ struct cxl_dev_id *dev_id;
+
+ /* dev_id must be globally unique and must contain the cxlds */
+ dev_id = devm_kzalloc(dev, sizeof(*dev_id), GFP_KERNEL);
+ if (!dev_id)
+ return -ENOMEM;
+ dev_id->cxlds = cxlds;
+
+ return devm_request_threaded_irq(dev, irq, handler, thread_fn,
+ IRQF_SHARED | IRQF_ONESHOT,
+ NULL, dev_id);
+}
+
+static bool cxl_mbox_background_complete(struct cxl_dev_state *cxlds)
+{
+ u64 reg;
+
+ reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_BG_CMD_STATUS_OFFSET);
+ return FIELD_GET(CXLDEV_MBOX_BG_CMD_COMMAND_PCT_MASK, reg) == 100;
+}
+
+static irqreturn_t cxl_pci_mbox_irq(int irq, void *id)
+{
+ u64 reg;
+ u16 opcode;
+ struct cxl_dev_id *dev_id = id;
+ struct cxl_dev_state *cxlds = dev_id->cxlds;
+ struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
+
+ if (!cxl_mbox_background_complete(cxlds))
+ return IRQ_NONE;
+
+ reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_BG_CMD_STATUS_OFFSET);
+ opcode = FIELD_GET(CXLDEV_MBOX_BG_CMD_COMMAND_OPCODE_MASK, reg);
+ if (opcode == CXL_MBOX_OP_SANITIZE) {
+ if (mds->security.sanitize_node)
+ sysfs_notify_dirent(mds->security.sanitize_node);
+
+ dev_dbg(cxlds->dev, "Sanitization operation ended\n");
+ } else {
+ /* short-circuit the wait in __cxl_pci_mbox_send_cmd() */
+ rcuwait_wake_up(&mds->mbox_wait);
+ }
+
+ return IRQ_HANDLED;
+}
+
+/*
+ * Sanitization operation polling mode.
+ */
+static void cxl_mbox_sanitize_work(struct work_struct *work)
+{
+ struct cxl_memdev_state *mds =
+ container_of(work, typeof(*mds), security.poll_dwork.work);
+ struct cxl_dev_state *cxlds = &mds->cxlds;
+
+ mutex_lock(&mds->mbox_mutex);
+ if (cxl_mbox_background_complete(cxlds)) {
+ mds->security.poll_tmo_secs = 0;
+ put_device(cxlds->dev);
+
+ if (mds->security.sanitize_node)
+ sysfs_notify_dirent(mds->security.sanitize_node);
+
+ dev_dbg(cxlds->dev, "Sanitization operation ended\n");
+ } else {
+ int timeout = mds->security.poll_tmo_secs + 10;
+
+ mds->security.poll_tmo_secs = min(15 * 60, timeout);
+ queue_delayed_work(system_wq, &mds->security.poll_dwork,
+ timeout * HZ);
+ }
+ mutex_unlock(&mds->mbox_mutex);
+}
+
/**
* __cxl_pci_mbox_send_cmd() - Execute a mailbox command
- * @cxlds: The device state to communicate with.
+ * @mds: The memory device driver data
* @mbox_cmd: Command to send to the memory device.
*
* Context: Any context. Expects mbox_mutex to be held.
@@ -106,16 +190,17 @@ static int cxl_pci_mbox_wait_for_doorbell(struct cxl_dev_state *cxlds)
* not need to coordinate with each other. The driver only uses the primary
* mailbox.
*/
-static int __cxl_pci_mbox_send_cmd(struct cxl_dev_state *cxlds,
+static int __cxl_pci_mbox_send_cmd(struct cxl_memdev_state *mds,
struct cxl_mbox_cmd *mbox_cmd)
{
+ struct cxl_dev_state *cxlds = &mds->cxlds;
void __iomem *payload = cxlds->regs.mbox + CXLDEV_MBOX_PAYLOAD_OFFSET;
struct device *dev = cxlds->dev;
u64 cmd_reg, status_reg;
size_t out_len;
int rc;
- lockdep_assert_held(&cxlds->mbox_mutex);
+ lockdep_assert_held(&mds->mbox_mutex);
/*
* Here are the steps from 8.2.8.4 of the CXL 2.0 spec.
@@ -144,6 +229,16 @@ static int __cxl_pci_mbox_send_cmd(struct cxl_dev_state *cxlds,
return -EBUSY;
}
+ /*
+ * With sanitize polling, hardware might be done and the poller still
+ * not be in sync. Ensure no new command comes in until so. Keep the
+ * hardware semantics and only allow device health status.
+ */
+ if (mds->security.poll_tmo_secs > 0) {
+ if (mbox_cmd->opcode != CXL_MBOX_OP_GET_HEALTH_INFO)
+ return -EBUSY;
+ }
+
cmd_reg = FIELD_PREP(CXLDEV_MBOX_CMD_COMMAND_OPCODE_MASK,
mbox_cmd->opcode);
if (mbox_cmd->size_in) {
@@ -177,12 +272,80 @@ static int __cxl_pci_mbox_send_cmd(struct cxl_dev_state *cxlds,
mbox_cmd->return_code =
FIELD_GET(CXLDEV_MBOX_STATUS_RET_CODE_MASK, status_reg);
+ /*
+ * Handle the background command in a synchronous manner.
+ *
+ * All other mailbox commands will serialize/queue on the mbox_mutex,
+ * which we currently hold. Furthermore this also guarantees that
+ * cxl_mbox_background_complete() checks are safe amongst each other,
+ * in that no new bg operation can occur in between.
+ *
+ * Background operations are timesliced in accordance with the nature
+ * of the command. In the event of timeout, the mailbox state is
+ * indeterminate until the next successful command submission and the
+ * driver can get back in sync with the hardware state.
+ */
+ if (mbox_cmd->return_code == CXL_MBOX_CMD_RC_BACKGROUND) {
+ u64 bg_status_reg;
+ int i, timeout;
+
+ /*
+ * Sanitization is a special case which monopolizes the device
+ * and cannot be timesliced. Handle asynchronously instead,
+ * and allow userspace to poll(2) for completion.
+ */
+ if (mbox_cmd->opcode == CXL_MBOX_OP_SANITIZE) {
+ if (mds->security.poll) {
+ /* hold the device throughout */
+ get_device(cxlds->dev);
+
+ /* give first timeout a second */
+ timeout = 1;
+ mds->security.poll_tmo_secs = timeout;
+ queue_delayed_work(system_wq,
+ &mds->security.poll_dwork,
+ timeout * HZ);
+ }
+
+ dev_dbg(dev, "Sanitization operation started\n");
+ goto success;
+ }
+
+ dev_dbg(dev, "Mailbox background operation (0x%04x) started\n",
+ mbox_cmd->opcode);
+
+ timeout = mbox_cmd->poll_interval_ms;
+ for (i = 0; i < mbox_cmd->poll_count; i++) {
+ if (rcuwait_wait_event_timeout(&mds->mbox_wait,
+ cxl_mbox_background_complete(cxlds),
+ TASK_UNINTERRUPTIBLE,
+ msecs_to_jiffies(timeout)) > 0)
+ break;
+ }
+
+ if (!cxl_mbox_background_complete(cxlds)) {
+ dev_err(dev, "timeout waiting for background (%d ms)\n",
+ timeout * mbox_cmd->poll_count);
+ return -ETIMEDOUT;
+ }
+
+ bg_status_reg = readq(cxlds->regs.mbox +
+ CXLDEV_MBOX_BG_CMD_STATUS_OFFSET);
+ mbox_cmd->return_code =
+ FIELD_GET(CXLDEV_MBOX_BG_CMD_COMMAND_RC_MASK,
+ bg_status_reg);
+ dev_dbg(dev,
+ "Mailbox background operation (0x%04x) completed\n",
+ mbox_cmd->opcode);
+ }
+
if (mbox_cmd->return_code != CXL_MBOX_CMD_RC_SUCCESS) {
dev_dbg(dev, "Mailbox operation had an error: %s\n",
cxl_mbox_cmd_rc2str(mbox_cmd));
return 0; /* completed but caller must check return_code */
}
+success:
/* #7 */
cmd_reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_CMD_OFFSET);
out_len = FIELD_GET(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, cmd_reg);
@@ -196,8 +359,9 @@ static int __cxl_pci_mbox_send_cmd(struct cxl_dev_state *cxlds,
* have requested less data than the hardware supplied even
* within spec.
*/
- size_t n = min3(mbox_cmd->size_out, cxlds->payload_size, out_len);
+ size_t n;
+ n = min3(mbox_cmd->size_out, mds->payload_size, out_len);
memcpy_fromio(mbox_cmd->payload_out, payload, n);
mbox_cmd->size_out = n;
} else {
@@ -207,20 +371,23 @@ static int __cxl_pci_mbox_send_cmd(struct cxl_dev_state *cxlds,
return 0;
}
-static int cxl_pci_mbox_send(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd *cmd)
+static int cxl_pci_mbox_send(struct cxl_memdev_state *mds,
+ struct cxl_mbox_cmd *cmd)
{
int rc;
- mutex_lock_io(&cxlds->mbox_mutex);
- rc = __cxl_pci_mbox_send_cmd(cxlds, cmd);
- mutex_unlock(&cxlds->mbox_mutex);
+ mutex_lock_io(&mds->mbox_mutex);
+ rc = __cxl_pci_mbox_send_cmd(mds, cmd);
+ mutex_unlock(&mds->mbox_mutex);
return rc;
}
-static int cxl_pci_setup_mailbox(struct cxl_dev_state *cxlds)
+static int cxl_pci_setup_mailbox(struct cxl_memdev_state *mds)
{
+ struct cxl_dev_state *cxlds = &mds->cxlds;
const int cap = readl(cxlds->regs.mbox + CXLDEV_MBOX_CAPS_OFFSET);
+ struct device *dev = cxlds->dev;
unsigned long timeout;
u64 md_status;
@@ -234,8 +401,7 @@ static int cxl_pci_setup_mailbox(struct cxl_dev_state *cxlds)
} while (!time_after(jiffies, timeout));
if (!(md_status & CXLMDEV_MBOX_IF_READY)) {
- cxl_err(cxlds->dev, md_status,
- "timeout awaiting mailbox ready");
+ cxl_err(dev, md_status, "timeout awaiting mailbox ready");
return -ETIMEDOUT;
}
@@ -246,12 +412,12 @@ static int cxl_pci_setup_mailbox(struct cxl_dev_state *cxlds)
* source for future doorbell busy events.
*/
if (cxl_pci_mbox_wait_for_doorbell(cxlds) != 0) {
- cxl_err(cxlds->dev, md_status, "timeout awaiting mailbox idle");
+ cxl_err(dev, md_status, "timeout awaiting mailbox idle");
return -ETIMEDOUT;
}
- cxlds->mbox_send = cxl_pci_mbox_send;
- cxlds->payload_size =
+ mds->mbox_send = cxl_pci_mbox_send;
+ mds->payload_size =
1 << FIELD_GET(CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK, cap);
/*
@@ -261,101 +427,46 @@ static int cxl_pci_setup_mailbox(struct cxl_dev_state *cxlds)
* there's no point in going forward. If the size is too large, there's
* no harm is soft limiting it.
*/
- cxlds->payload_size = min_t(size_t, cxlds->payload_size, SZ_1M);
- if (cxlds->payload_size < 256) {
- dev_err(cxlds->dev, "Mailbox is too small (%zub)",
- cxlds->payload_size);
+ mds->payload_size = min_t(size_t, mds->payload_size, SZ_1M);
+ if (mds->payload_size < 256) {
+ dev_err(dev, "Mailbox is too small (%zub)",
+ mds->payload_size);
return -ENXIO;
}
- dev_dbg(cxlds->dev, "Mailbox payload sized %zu",
- cxlds->payload_size);
+ dev_dbg(dev, "Mailbox payload sized %zu", mds->payload_size);
- return 0;
-}
+ rcuwait_init(&mds->mbox_wait);
-static int cxl_map_regblock(struct pci_dev *pdev, struct cxl_register_map *map)
-{
- struct device *dev = &pdev->dev;
+ if (cap & CXLDEV_MBOX_CAP_BG_CMD_IRQ) {
+ u32 ctrl;
+ int irq, msgnum;
+ struct pci_dev *pdev = to_pci_dev(cxlds->dev);
- map->base = ioremap(map->resource, map->max_size);
- if (!map->base) {
- dev_err(dev, "failed to map registers\n");
- return -ENOMEM;
+ msgnum = FIELD_GET(CXLDEV_MBOX_CAP_IRQ_MSGNUM_MASK, cap);
+ irq = pci_irq_vector(pdev, msgnum);
+ if (irq < 0)
+ goto mbox_poll;
+
+ if (cxl_request_irq(cxlds, irq, cxl_pci_mbox_irq, NULL))
+ goto mbox_poll;
+
+ /* enable background command mbox irq support */
+ ctrl = readl(cxlds->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET);
+ ctrl |= CXLDEV_MBOX_CTRL_BG_CMD_IRQ;
+ writel(ctrl, cxlds->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET);
+
+ return 0;
}
- dev_dbg(dev, "Mapped CXL Memory Device resource %pa\n", &map->resource);
+mbox_poll:
+ mds->security.poll = true;
+ INIT_DELAYED_WORK(&mds->security.poll_dwork, cxl_mbox_sanitize_work);
+
+ dev_dbg(cxlds->dev, "Mailbox interrupts are unsupported");
return 0;
}
-static void cxl_unmap_regblock(struct pci_dev *pdev,
- struct cxl_register_map *map)
-{
- iounmap(map->base);
- map->base = NULL;
-}
-
-static int cxl_probe_regs(struct pci_dev *pdev, struct cxl_register_map *map)
-{
- struct cxl_component_reg_map *comp_map;
- struct cxl_device_reg_map *dev_map;
- struct device *dev = &pdev->dev;
- void __iomem *base = map->base;
-
- switch (map->reg_type) {
- case CXL_REGLOC_RBI_COMPONENT:
- comp_map = &map->component_map;
- cxl_probe_component_regs(dev, base, comp_map);
- if (!comp_map->hdm_decoder.valid) {
- dev_err(dev, "HDM decoder registers not found\n");
- return -ENXIO;
- }
-
- if (!comp_map->ras.valid)
- dev_dbg(dev, "RAS registers not found\n");
-
- dev_dbg(dev, "Set up component registers\n");
- break;
- case CXL_REGLOC_RBI_MEMDEV:
- dev_map = &map->device_map;
- cxl_probe_device_regs(dev, base, dev_map);
- if (!dev_map->status.valid || !dev_map->mbox.valid ||
- !dev_map->memdev.valid) {
- dev_err(dev, "registers not found: %s%s%s\n",
- !dev_map->status.valid ? "status " : "",
- !dev_map->mbox.valid ? "mbox " : "",
- !dev_map->memdev.valid ? "memdev " : "");
- return -ENXIO;
- }
-
- dev_dbg(dev, "Probing device registers...\n");
- break;
- default:
- break;
- }
-
- return 0;
-}
-
-static int cxl_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
- struct cxl_register_map *map)
-{
- int rc;
-
- rc = cxl_find_regblock(pdev, type, map);
- if (rc)
- return rc;
-
- rc = cxl_map_regblock(pdev, map);
- if (rc)
- return rc;
-
- rc = cxl_probe_regs(pdev, map);
- cxl_unmap_regblock(pdev, map);
-
- return rc;
-}
-
/*
* Assume that any RCIEP that emits the CXL memory expander class code
* is an RCD
@@ -365,17 +476,55 @@ static bool is_cxl_restricted(struct pci_dev *pdev)
return pci_pcie_type(pdev) == PCI_EXP_TYPE_RC_END;
}
-/*
- * CXL v3.0 6.2.3 Table 6-4
- * The table indicates that if PCIe Flit Mode is set, then CXL is in 256B flits
- * mode, otherwise it's 68B flits mode.
- */
-static bool cxl_pci_flit_256(struct pci_dev *pdev)
+static int cxl_rcrb_get_comp_regs(struct pci_dev *pdev,
+ struct cxl_register_map *map)
{
- u16 lnksta2;
+ struct cxl_port *port;
+ struct cxl_dport *dport;
+ resource_size_t component_reg_phys;
- pcie_capability_read_word(pdev, PCI_EXP_LNKSTA2, &lnksta2);
- return lnksta2 & PCI_EXP_LNKSTA2_FLIT;
+ *map = (struct cxl_register_map) {
+ .dev = &pdev->dev,
+ .resource = CXL_RESOURCE_NONE,
+ };
+
+ port = cxl_pci_find_port(pdev, &dport);
+ if (!port)
+ return -EPROBE_DEFER;
+
+ component_reg_phys = cxl_rcd_component_reg_phys(&pdev->dev, dport);
+
+ put_device(&port->dev);
+
+ if (component_reg_phys == CXL_RESOURCE_NONE)
+ return -ENXIO;
+
+ map->resource = component_reg_phys;
+ map->reg_type = CXL_REGLOC_RBI_COMPONENT;
+ map->max_size = CXL_COMPONENT_REG_BLOCK_SIZE;
+
+ return 0;
+}
+
+static int cxl_pci_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
+ struct cxl_register_map *map)
+{
+ int rc;
+
+ rc = cxl_find_regblock(pdev, type, map);
+
+ /*
+ * If the Register Locator DVSEC does not exist, check if it
+ * is an RCH and try to extract the Component Registers from
+ * an RCRB.
+ */
+ if (rc && type == CXL_REGLOC_RBI_COMPONENT && is_cxl_restricted(pdev))
+ rc = cxl_rcrb_get_comp_regs(pdev, map);
+
+ if (rc)
+ return rc;
+
+ return cxl_setup_regs(map);
}
static int cxl_pci_ras_unmask(struct pci_dev *pdev)
@@ -404,9 +553,8 @@ static int cxl_pci_ras_unmask(struct pci_dev *pdev)
addr = cxlds->regs.ras + CXL_RAS_UNCORRECTABLE_MASK_OFFSET;
orig_val = readl(addr);
- mask = CXL_RAS_UNCORRECTABLE_MASK_MASK;
- if (!cxl_pci_flit_256(pdev))
- mask &= ~CXL_RAS_UNCORRECTABLE_MASK_F256B_MASK;
+ mask = CXL_RAS_UNCORRECTABLE_MASK_MASK |
+ CXL_RAS_UNCORRECTABLE_MASK_F256B_MASK;
val = orig_val & ~mask;
writel(val, addr);
dev_dbg(&pdev->dev,
@@ -433,18 +581,18 @@ static void free_event_buf(void *buf)
/*
* There is a single buffer for reading event logs from the mailbox. All logs
- * share this buffer protected by the cxlds->event_log_lock.
+ * share this buffer protected by the mds->event_log_lock.
*/
-static int cxl_mem_alloc_event_buf(struct cxl_dev_state *cxlds)
+static int cxl_mem_alloc_event_buf(struct cxl_memdev_state *mds)
{
struct cxl_get_event_payload *buf;
- buf = kvmalloc(cxlds->payload_size, GFP_KERNEL);
+ buf = kvmalloc(mds->payload_size, GFP_KERNEL);
if (!buf)
return -ENOMEM;
- cxlds->event.buf = buf;
+ mds->event.buf = buf;
- return devm_add_action_or_reset(cxlds->dev, free_event_buf, buf);
+ return devm_add_action_or_reset(mds->cxlds.dev, free_event_buf, buf);
}
static int cxl_alloc_irq_vectors(struct pci_dev *pdev)
@@ -469,14 +617,11 @@ static int cxl_alloc_irq_vectors(struct pci_dev *pdev)
return 0;
}
-struct cxl_dev_id {
- struct cxl_dev_state *cxlds;
-};
-
static irqreturn_t cxl_event_thread(int irq, void *id)
{
struct cxl_dev_id *dev_id = id;
struct cxl_dev_state *cxlds = dev_id->cxlds;
+ struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
u32 status;
do {
@@ -489,7 +634,7 @@ static irqreturn_t cxl_event_thread(int irq, void *id)
status &= CXLDEV_EVENT_STATUS_ALL;
if (!status)
break;
- cxl_mem_get_event_records(cxlds, status);
+ cxl_mem_get_event_records(mds, status);
cond_resched();
} while (status);
@@ -498,31 +643,21 @@ static irqreturn_t cxl_event_thread(int irq, void *id)
static int cxl_event_req_irq(struct cxl_dev_state *cxlds, u8 setting)
{
- struct device *dev = cxlds->dev;
- struct pci_dev *pdev = to_pci_dev(dev);
- struct cxl_dev_id *dev_id;
+ struct pci_dev *pdev = to_pci_dev(cxlds->dev);
int irq;
if (FIELD_GET(CXLDEV_EVENT_INT_MODE_MASK, setting) != CXL_INT_MSI_MSIX)
return -ENXIO;
- /* dev_id must be globally unique and must contain the cxlds */
- dev_id = devm_kzalloc(dev, sizeof(*dev_id), GFP_KERNEL);
- if (!dev_id)
- return -ENOMEM;
- dev_id->cxlds = cxlds;
-
irq = pci_irq_vector(pdev,
FIELD_GET(CXLDEV_EVENT_INT_MSGNUM_MASK, setting));
if (irq < 0)
return irq;
- return devm_request_threaded_irq(dev, irq, NULL, cxl_event_thread,
- IRQF_SHARED | IRQF_ONESHOT, NULL,
- dev_id);
+ return cxl_request_irq(cxlds, irq, NULL, cxl_event_thread);
}
-static int cxl_event_get_int_policy(struct cxl_dev_state *cxlds,
+static int cxl_event_get_int_policy(struct cxl_memdev_state *mds,
struct cxl_event_interrupt_policy *policy)
{
struct cxl_mbox_cmd mbox_cmd = {
@@ -532,15 +667,15 @@ static int cxl_event_get_int_policy(struct cxl_dev_state *cxlds,
};
int rc;
- rc = cxl_internal_send_cmd(cxlds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(mds, &mbox_cmd);
if (rc < 0)
- dev_err(cxlds->dev, "Failed to get event interrupt policy : %d",
- rc);
+ dev_err(mds->cxlds.dev,
+ "Failed to get event interrupt policy : %d", rc);
return rc;
}
-static int cxl_event_config_msgnums(struct cxl_dev_state *cxlds,
+static int cxl_event_config_msgnums(struct cxl_memdev_state *mds,
struct cxl_event_interrupt_policy *policy)
{
struct cxl_mbox_cmd mbox_cmd;
@@ -559,23 +694,24 @@ static int cxl_event_config_msgnums(struct cxl_dev_state *cxlds,
.size_in = sizeof(*policy),
};
- rc = cxl_internal_send_cmd(cxlds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(mds, &mbox_cmd);
if (rc < 0) {
- dev_err(cxlds->dev, "Failed to set event interrupt policy : %d",
+ dev_err(mds->cxlds.dev, "Failed to set event interrupt policy : %d",
rc);
return rc;
}
/* Retrieve final interrupt settings */
- return cxl_event_get_int_policy(cxlds, policy);
+ return cxl_event_get_int_policy(mds, policy);
}
-static int cxl_event_irqsetup(struct cxl_dev_state *cxlds)
+static int cxl_event_irqsetup(struct cxl_memdev_state *mds)
{
+ struct cxl_dev_state *cxlds = &mds->cxlds;
struct cxl_event_interrupt_policy policy;
int rc;
- rc = cxl_event_config_msgnums(cxlds, &policy);
+ rc = cxl_event_config_msgnums(mds, &policy);
if (rc)
return rc;
@@ -614,7 +750,7 @@ static bool cxl_event_int_is_fw(u8 setting)
}
static int cxl_event_config(struct pci_host_bridge *host_bridge,
- struct cxl_dev_state *cxlds)
+ struct cxl_memdev_state *mds)
{
struct cxl_event_interrupt_policy policy;
int rc;
@@ -626,11 +762,11 @@ static int cxl_event_config(struct pci_host_bridge *host_bridge,
if (!host_bridge->native_cxl_error)
return 0;
- rc = cxl_mem_alloc_event_buf(cxlds);
+ rc = cxl_mem_alloc_event_buf(mds);
if (rc)
return rc;
- rc = cxl_event_get_int_policy(cxlds, &policy);
+ rc = cxl_event_get_int_policy(mds, &policy);
if (rc)
return rc;
@@ -638,15 +774,16 @@ static int cxl_event_config(struct pci_host_bridge *host_bridge,
cxl_event_int_is_fw(policy.warn_settings) ||
cxl_event_int_is_fw(policy.failure_settings) ||
cxl_event_int_is_fw(policy.fatal_settings)) {
- dev_err(cxlds->dev, "FW still in control of Event Logs despite _OSC settings\n");
+ dev_err(mds->cxlds.dev,
+ "FW still in control of Event Logs despite _OSC settings\n");
return -EBUSY;
}
- rc = cxl_event_irqsetup(cxlds);
+ rc = cxl_event_irqsetup(mds);
if (rc)
return rc;
- cxl_mem_get_event_records(cxlds, CXLDEV_EVENT_STATUS_ALL);
+ cxl_mem_get_event_records(mds, CXLDEV_EVENT_STATUS_ALL);
return 0;
}
@@ -654,10 +791,11 @@ static int cxl_event_config(struct pci_host_bridge *host_bridge,
static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
struct pci_host_bridge *host_bridge = pci_find_host_bridge(pdev->bus);
+ struct cxl_memdev_state *mds;
+ struct cxl_dev_state *cxlds;
struct cxl_register_map map;
struct cxl_memdev *cxlmd;
- struct cxl_dev_state *cxlds;
- int rc;
+ int i, rc, pmu_count;
/*
* Double check the anonymous union trickery in struct cxl_regs
@@ -671,9 +809,10 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
return rc;
pci_set_master(pdev);
- cxlds = cxl_dev_state_create(&pdev->dev);
- if (IS_ERR(cxlds))
- return PTR_ERR(cxlds);
+ mds = cxl_memdev_state_create(&pdev->dev);
+ if (IS_ERR(mds))
+ return PTR_ERR(mds);
+ cxlds = &mds->cxlds;
pci_set_drvdata(pdev, cxlds);
cxlds->rcd = is_cxl_restricted(pdev);
@@ -684,11 +823,11 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
dev_warn(&pdev->dev,
"Device DVSEC not present, skip CXL.mem init\n");
- rc = cxl_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map);
+ rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map);
if (rc)
return rc;
- rc = cxl_map_device_regs(&pdev->dev, &cxlds->regs.device_regs, &map);
+ rc = cxl_map_device_regs(&map, &cxlds->regs.device_regs);
if (rc)
return rc;
@@ -697,14 +836,16 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
* still be useful for management functions so don't return an error.
*/
cxlds->component_reg_phys = CXL_RESOURCE_NONE;
- rc = cxl_setup_regs(pdev, CXL_REGLOC_RBI_COMPONENT, &map);
+ rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_COMPONENT, &map);
if (rc)
dev_warn(&pdev->dev, "No component registers (%d)\n", rc);
+ else if (!map.component_map.ras.valid)
+ dev_dbg(&pdev->dev, "RAS registers not found\n");
cxlds->component_reg_phys = map.resource;
- rc = cxl_map_component_regs(&pdev->dev, &cxlds->regs.component,
- &map, BIT(CXL_CM_CAP_CAP_ID_RAS));
+ rc = cxl_map_component_regs(&map, &cxlds->regs.component,
+ BIT(CXL_CM_CAP_CAP_ID_RAS));
if (rc)
dev_dbg(&pdev->dev, "Failed to map RAS capability.\n");
@@ -714,39 +855,66 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
else
dev_warn(&pdev->dev, "Media not active (%d)\n", rc);
- rc = cxl_pci_setup_mailbox(cxlds);
- if (rc)
- return rc;
-
- rc = cxl_enumerate_cmds(cxlds);
- if (rc)
- return rc;
-
- rc = cxl_set_timestamp(cxlds);
- if (rc)
- return rc;
-
- rc = cxl_poison_state_init(cxlds);
- if (rc)
- return rc;
-
- rc = cxl_dev_state_identify(cxlds);
- if (rc)
- return rc;
-
- rc = cxl_mem_create_range_info(cxlds);
- if (rc)
- return rc;
-
rc = cxl_alloc_irq_vectors(pdev);
if (rc)
return rc;
+ rc = cxl_pci_setup_mailbox(mds);
+ if (rc)
+ return rc;
+
+ rc = cxl_enumerate_cmds(mds);
+ if (rc)
+ return rc;
+
+ rc = cxl_set_timestamp(mds);
+ if (rc)
+ return rc;
+
+ rc = cxl_poison_state_init(mds);
+ if (rc)
+ return rc;
+
+ rc = cxl_dev_state_identify(mds);
+ if (rc)
+ return rc;
+
+ rc = cxl_mem_create_range_info(mds);
+ if (rc)
+ return rc;
+
cxlmd = devm_cxl_add_memdev(cxlds);
if (IS_ERR(cxlmd))
return PTR_ERR(cxlmd);
- rc = cxl_event_config(host_bridge, cxlds);
+ rc = cxl_memdev_setup_fw_upload(mds);
+ if (rc)
+ return rc;
+
+ pmu_count = cxl_count_regblock(pdev, CXL_REGLOC_RBI_PMU);
+ for (i = 0; i < pmu_count; i++) {
+ struct cxl_pmu_regs pmu_regs;
+
+ rc = cxl_find_regblock_instance(pdev, CXL_REGLOC_RBI_PMU, &map, i);
+ if (rc) {
+ dev_dbg(&pdev->dev, "Could not find PMU regblock\n");
+ break;
+ }
+
+ rc = cxl_map_pmu_regs(pdev, &pmu_regs, &map);
+ if (rc) {
+ dev_dbg(&pdev->dev, "Could not map PMU regs\n");
+ break;
+ }
+
+ rc = devm_cxl_pmu_add(cxlds->dev, &pmu_regs, cxlmd->id, i, CXL_PMU_MEMDEV);
+ if (rc) {
+ dev_dbg(&pdev->dev, "Could not add PMU instance\n");
+ break;
+ }
+ }
+
+ rc = cxl_event_config(host_bridge, mds);
if (rc)
return rc;
diff --git a/drivers/cxl/pmem.c b/drivers/cxl/pmem.c
index 71cfa1f..7cb8994 100644
--- a/drivers/cxl/pmem.c
+++ b/drivers/cxl/pmem.c
@@ -15,9 +15,9 @@ extern const struct nvdimm_security_ops *cxl_security_ops;
static __read_mostly DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX);
-static void clear_exclusive(void *cxlds)
+static void clear_exclusive(void *mds)
{
- clear_exclusive_cxl_commands(cxlds, exclusive_cmds);
+ clear_exclusive_cxl_commands(mds, exclusive_cmds);
}
static void unregister_nvdimm(void *nvdimm)
@@ -65,13 +65,13 @@ static int cxl_nvdimm_probe(struct device *dev)
struct cxl_nvdimm *cxl_nvd = to_cxl_nvdimm(dev);
struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
struct cxl_nvdimm_bridge *cxl_nvb = cxlmd->cxl_nvb;
+ struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
unsigned long flags = 0, cmd_mask = 0;
- struct cxl_dev_state *cxlds = cxlmd->cxlds;
struct nvdimm *nvdimm;
int rc;
- set_exclusive_cxl_commands(cxlds, exclusive_cmds);
- rc = devm_add_action_or_reset(dev, clear_exclusive, cxlds);
+ set_exclusive_cxl_commands(mds, exclusive_cmds);
+ rc = devm_add_action_or_reset(dev, clear_exclusive, mds);
if (rc)
return rc;
@@ -100,22 +100,23 @@ static struct cxl_driver cxl_nvdimm_driver = {
},
};
-static int cxl_pmem_get_config_size(struct cxl_dev_state *cxlds,
+static int cxl_pmem_get_config_size(struct cxl_memdev_state *mds,
struct nd_cmd_get_config_size *cmd,
unsigned int buf_len)
{
if (sizeof(*cmd) > buf_len)
return -EINVAL;
- *cmd = (struct nd_cmd_get_config_size) {
- .config_size = cxlds->lsa_size,
- .max_xfer = cxlds->payload_size - sizeof(struct cxl_mbox_set_lsa),
+ *cmd = (struct nd_cmd_get_config_size){
+ .config_size = mds->lsa_size,
+ .max_xfer =
+ mds->payload_size - sizeof(struct cxl_mbox_set_lsa),
};
return 0;
}
-static int cxl_pmem_get_config_data(struct cxl_dev_state *cxlds,
+static int cxl_pmem_get_config_data(struct cxl_memdev_state *mds,
struct nd_cmd_get_config_data_hdr *cmd,
unsigned int buf_len)
{
@@ -140,13 +141,13 @@ static int cxl_pmem_get_config_data(struct cxl_dev_state *cxlds,
.payload_out = cmd->out_buf,
};
- rc = cxl_internal_send_cmd(cxlds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(mds, &mbox_cmd);
cmd->status = 0;
return rc;
}
-static int cxl_pmem_set_config_data(struct cxl_dev_state *cxlds,
+static int cxl_pmem_set_config_data(struct cxl_memdev_state *mds,
struct nd_cmd_set_config_hdr *cmd,
unsigned int buf_len)
{
@@ -176,7 +177,7 @@ static int cxl_pmem_set_config_data(struct cxl_dev_state *cxlds,
.size_in = struct_size(set_lsa, data, cmd->in_length),
};
- rc = cxl_internal_send_cmd(cxlds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(mds, &mbox_cmd);
/*
* Set "firmware" status (4-packed bytes at the end of the input
@@ -194,18 +195,18 @@ static int cxl_pmem_nvdimm_ctl(struct nvdimm *nvdimm, unsigned int cmd,
struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
unsigned long cmd_mask = nvdimm_cmd_mask(nvdimm);
struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
- struct cxl_dev_state *cxlds = cxlmd->cxlds;
+ struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
if (!test_bit(cmd, &cmd_mask))
return -ENOTTY;
switch (cmd) {
case ND_CMD_GET_CONFIG_SIZE:
- return cxl_pmem_get_config_size(cxlds, buf, buf_len);
+ return cxl_pmem_get_config_size(mds, buf, buf_len);
case ND_CMD_GET_CONFIG_DATA:
- return cxl_pmem_get_config_data(cxlds, buf, buf_len);
+ return cxl_pmem_get_config_data(mds, buf, buf_len);
case ND_CMD_SET_CONFIG_DATA:
- return cxl_pmem_set_config_data(cxlds, buf, buf_len);
+ return cxl_pmem_set_config_data(mds, buf, buf_len);
default:
return -ENOTTY;
}
diff --git a/drivers/cxl/pmu.h b/drivers/cxl/pmu.h
new file mode 100644
index 0000000..b1e9bcd
--- /dev/null
+++ b/drivers/cxl/pmu.h
@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright(c) 2023 Huawei
+ * CXL Specification rev 3.0 Setion 8.2.7 (CPMU Register Interface)
+ */
+#ifndef CXL_PMU_H
+#define CXL_PMU_H
+#include <linux/device.h>
+
+enum cxl_pmu_type {
+ CXL_PMU_MEMDEV,
+};
+
+#define CXL_PMU_REGMAP_SIZE 0xe00 /* Table 8-32 CXL 3.0 specification */
+struct cxl_pmu {
+ struct device dev;
+ void __iomem *base;
+ int assoc_id;
+ int index;
+ enum cxl_pmu_type type;
+};
+
+#define to_cxl_pmu(dev) container_of(dev, struct cxl_pmu, dev)
+struct cxl_pmu_regs;
+int devm_cxl_pmu_add(struct device *parent, struct cxl_pmu_regs *regs,
+ int assoc_id, int idx, enum cxl_pmu_type type);
+
+#endif
diff --git a/drivers/cxl/port.c b/drivers/cxl/port.c
index c23b616..6240e05 100644
--- a/drivers/cxl/port.c
+++ b/drivers/cxl/port.c
@@ -60,17 +60,13 @@ static int discover_region(struct device *dev, void *root)
static int cxl_switch_port_probe(struct cxl_port *port)
{
struct cxl_hdm *cxlhdm;
- int rc, nr_dports;
+ int rc;
- nr_dports = devm_cxl_port_enumerate_dports(port);
- if (nr_dports < 0)
- return nr_dports;
-
- cxlhdm = devm_cxl_setup_hdm(port, NULL);
- rc = devm_cxl_enable_hdm(port, cxlhdm);
- if (rc)
+ rc = devm_cxl_port_enumerate_dports(port);
+ if (rc < 0)
return rc;
+ cxlhdm = devm_cxl_setup_hdm(port, NULL);
if (!IS_ERR(cxlhdm))
return devm_cxl_enumerate_decoders(cxlhdm, NULL);
@@ -79,7 +75,7 @@ static int cxl_switch_port_probe(struct cxl_port *port)
return PTR_ERR(cxlhdm);
}
- if (nr_dports == 1) {
+ if (rc == 1) {
dev_dbg(&port->dev, "Fallback to passthrough decoder\n");
return devm_cxl_add_passthrough_decoder(port);
}
@@ -91,7 +87,7 @@ static int cxl_switch_port_probe(struct cxl_port *port)
static int cxl_endpoint_port_probe(struct cxl_port *port)
{
struct cxl_endpoint_dvsec_info info = { .port = port };
- struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport);
+ struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev);
struct cxl_dev_state *cxlds = cxlmd->cxlds;
struct cxl_hdm *cxlhdm;
struct cxl_port *root;
@@ -102,8 +98,11 @@ static int cxl_endpoint_port_probe(struct cxl_port *port)
return rc;
cxlhdm = devm_cxl_setup_hdm(port, &info);
- if (IS_ERR(cxlhdm))
+ if (IS_ERR(cxlhdm)) {
+ if (PTR_ERR(cxlhdm) == -ENODEV)
+ dev_err(&port->dev, "HDM decoder registers not found\n");
return PTR_ERR(cxlhdm);
+ }
/* Cache the data early to ensure is_visible() works */
read_cdat_data(port);
diff --git a/drivers/cxl/security.c b/drivers/cxl/security.c
index 4ad4bda..21856a3 100644
--- a/drivers/cxl/security.c
+++ b/drivers/cxl/security.c
@@ -14,7 +14,7 @@ static unsigned long cxl_pmem_get_security_flags(struct nvdimm *nvdimm,
{
struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
- struct cxl_dev_state *cxlds = cxlmd->cxlds;
+ struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
unsigned long security_flags = 0;
struct cxl_get_security_output {
__le32 flags;
@@ -29,11 +29,14 @@ static unsigned long cxl_pmem_get_security_flags(struct nvdimm *nvdimm,
.payload_out = &out,
};
- rc = cxl_internal_send_cmd(cxlds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(mds, &mbox_cmd);
if (rc < 0)
return 0;
sec_out = le32_to_cpu(out.flags);
+ /* cache security state */
+ mds->security.state = sec_out;
+
if (ptype == NVDIMM_MASTER) {
if (sec_out & CXL_PMEM_SEC_STATE_MASTER_PASS_SET)
set_bit(NVDIMM_SECURITY_UNLOCKED, &security_flags);
@@ -67,7 +70,7 @@ static int cxl_pmem_security_change_key(struct nvdimm *nvdimm,
{
struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
- struct cxl_dev_state *cxlds = cxlmd->cxlds;
+ struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
struct cxl_mbox_cmd mbox_cmd;
struct cxl_set_pass set_pass;
@@ -84,7 +87,7 @@ static int cxl_pmem_security_change_key(struct nvdimm *nvdimm,
.payload_in = &set_pass,
};
- return cxl_internal_send_cmd(cxlds, &mbox_cmd);
+ return cxl_internal_send_cmd(mds, &mbox_cmd);
}
static int __cxl_pmem_security_disable(struct nvdimm *nvdimm,
@@ -93,7 +96,7 @@ static int __cxl_pmem_security_disable(struct nvdimm *nvdimm,
{
struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
- struct cxl_dev_state *cxlds = cxlmd->cxlds;
+ struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
struct cxl_disable_pass dis_pass;
struct cxl_mbox_cmd mbox_cmd;
@@ -109,7 +112,7 @@ static int __cxl_pmem_security_disable(struct nvdimm *nvdimm,
.payload_in = &dis_pass,
};
- return cxl_internal_send_cmd(cxlds, &mbox_cmd);
+ return cxl_internal_send_cmd(mds, &mbox_cmd);
}
static int cxl_pmem_security_disable(struct nvdimm *nvdimm,
@@ -128,12 +131,12 @@ static int cxl_pmem_security_freeze(struct nvdimm *nvdimm)
{
struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
- struct cxl_dev_state *cxlds = cxlmd->cxlds;
+ struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
struct cxl_mbox_cmd mbox_cmd = {
.opcode = CXL_MBOX_OP_FREEZE_SECURITY,
};
- return cxl_internal_send_cmd(cxlds, &mbox_cmd);
+ return cxl_internal_send_cmd(mds, &mbox_cmd);
}
static int cxl_pmem_security_unlock(struct nvdimm *nvdimm,
@@ -141,7 +144,7 @@ static int cxl_pmem_security_unlock(struct nvdimm *nvdimm,
{
struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
- struct cxl_dev_state *cxlds = cxlmd->cxlds;
+ struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
u8 pass[NVDIMM_PASSPHRASE_LEN];
struct cxl_mbox_cmd mbox_cmd;
int rc;
@@ -153,7 +156,7 @@ static int cxl_pmem_security_unlock(struct nvdimm *nvdimm,
.payload_in = pass,
};
- rc = cxl_internal_send_cmd(cxlds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(mds, &mbox_cmd);
if (rc < 0)
return rc;
@@ -166,7 +169,7 @@ static int cxl_pmem_security_passphrase_erase(struct nvdimm *nvdimm,
{
struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
- struct cxl_dev_state *cxlds = cxlmd->cxlds;
+ struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
struct cxl_mbox_cmd mbox_cmd;
struct cxl_pass_erase erase;
int rc;
@@ -182,7 +185,7 @@ static int cxl_pmem_security_passphrase_erase(struct nvdimm *nvdimm,
.payload_in = &erase,
};
- rc = cxl_internal_send_cmd(cxlds, &mbox_cmd);
+ rc = cxl_internal_send_cmd(mds, &mbox_cmd);
if (rc < 0)
return rc;
diff --git a/drivers/dax/bus.c b/drivers/dax/bus.c
index 2278000..0ee96e6 100644
--- a/drivers/dax/bus.c
+++ b/drivers/dax/bus.c
@@ -446,18 +446,33 @@ static void unregister_dev_dax(void *dev)
put_device(dev);
}
+static void dax_region_free(struct kref *kref)
+{
+ struct dax_region *dax_region;
+
+ dax_region = container_of(kref, struct dax_region, kref);
+ kfree(dax_region);
+}
+
+static void dax_region_put(struct dax_region *dax_region)
+{
+ kref_put(&dax_region->kref, dax_region_free);
+}
+
/* a return value >= 0 indicates this invocation invalidated the id */
static int __free_dev_dax_id(struct dev_dax *dev_dax)
{
- struct dax_region *dax_region = dev_dax->region;
struct device *dev = &dev_dax->dev;
+ struct dax_region *dax_region;
int rc = dev_dax->id;
device_lock_assert(dev);
- if (is_static(dax_region) || dev_dax->id < 0)
+ if (!dev_dax->dyn_id || dev_dax->id < 0)
return -1;
+ dax_region = dev_dax->region;
ida_free(&dax_region->ida, dev_dax->id);
+ dax_region_put(dax_region);
dev_dax->id = -1;
return rc;
}
@@ -473,6 +488,20 @@ static int free_dev_dax_id(struct dev_dax *dev_dax)
return rc;
}
+static int alloc_dev_dax_id(struct dev_dax *dev_dax)
+{
+ struct dax_region *dax_region = dev_dax->region;
+ int id;
+
+ id = ida_alloc(&dax_region->ida, GFP_KERNEL);
+ if (id < 0)
+ return id;
+ kref_get(&dax_region->kref);
+ dev_dax->dyn_id = true;
+ dev_dax->id = id;
+ return id;
+}
+
static ssize_t delete_store(struct device *dev, struct device_attribute *attr,
const char *buf, size_t len)
{
@@ -560,20 +589,6 @@ static const struct attribute_group *dax_region_attribute_groups[] = {
NULL,
};
-static void dax_region_free(struct kref *kref)
-{
- struct dax_region *dax_region;
-
- dax_region = container_of(kref, struct dax_region, kref);
- kfree(dax_region);
-}
-
-void dax_region_put(struct dax_region *dax_region)
-{
- kref_put(&dax_region->kref, dax_region_free);
-}
-EXPORT_SYMBOL_GPL(dax_region_put);
-
static void dax_region_unregister(void *region)
{
struct dax_region *dax_region = region;
@@ -625,7 +640,6 @@ struct dax_region *alloc_dax_region(struct device *parent, int region_id,
return NULL;
}
- kref_get(&dax_region->kref);
if (devm_add_action_or_reset(parent, dax_region_unregister, dax_region))
return NULL;
return dax_region;
@@ -635,10 +649,12 @@ EXPORT_SYMBOL_GPL(alloc_dax_region);
static void dax_mapping_release(struct device *dev)
{
struct dax_mapping *mapping = to_dax_mapping(dev);
- struct dev_dax *dev_dax = to_dev_dax(dev->parent);
+ struct device *parent = dev->parent;
+ struct dev_dax *dev_dax = to_dev_dax(parent);
ida_free(&dev_dax->ida, mapping->id);
kfree(mapping);
+ put_device(parent);
}
static void unregister_dax_mapping(void *data)
@@ -655,8 +671,7 @@ static void unregister_dax_mapping(void *data)
dev_dax->ranges[mapping->range_id].mapping = NULL;
mapping->range_id = -1;
- device_del(dev);
- put_device(dev);
+ device_unregister(dev);
}
static struct dev_dax_range *get_dax_range(struct device *dev)
@@ -778,6 +793,7 @@ static int devm_register_dax_mapping(struct dev_dax *dev_dax, int range_id)
dev = &mapping->dev;
device_initialize(dev);
dev->parent = &dev_dax->dev;
+ get_device(dev->parent);
dev->type = &dax_mapping_type;
dev_set_name(dev, "mapping%d", mapping->id);
rc = device_add(dev);
@@ -1295,12 +1311,10 @@ static const struct attribute_group *dax_attribute_groups[] = {
static void dev_dax_release(struct device *dev)
{
struct dev_dax *dev_dax = to_dev_dax(dev);
- struct dax_region *dax_region = dev_dax->region;
struct dax_device *dax_dev = dev_dax->dax_dev;
put_dax(dax_dev);
free_dev_dax_id(dev_dax);
- dax_region_put(dax_region);
kfree(dev_dax->pgmap);
kfree(dev_dax);
}
@@ -1324,6 +1338,7 @@ struct dev_dax *devm_create_dev_dax(struct dev_dax_data *data)
if (!dev_dax)
return ERR_PTR(-ENOMEM);
+ dev_dax->region = dax_region;
if (is_static(dax_region)) {
if (dev_WARN_ONCE(parent, data->id < 0,
"dynamic id specified to static region\n")) {
@@ -1339,13 +1354,11 @@ struct dev_dax *devm_create_dev_dax(struct dev_dax_data *data)
goto err_id;
}
- rc = ida_alloc(&dax_region->ida, GFP_KERNEL);
+ rc = alloc_dev_dax_id(dev_dax);
if (rc < 0)
goto err_id;
- dev_dax->id = rc;
}
- dev_dax->region = dax_region;
dev = &dev_dax->dev;
device_initialize(dev);
dev_set_name(dev, "dax%d.%d", dax_region->id, dev_dax->id);
@@ -1386,7 +1399,6 @@ struct dev_dax *devm_create_dev_dax(struct dev_dax_data *data)
dev_dax->target_node = dax_region->target_node;
dev_dax->align = dax_region->align;
ida_init(&dev_dax->ida);
- kref_get(&dax_region->kref);
inode = dax_inode(dax_dev);
dev->devt = inode->i_rdev;
diff --git a/drivers/dax/bus.h b/drivers/dax/bus.h
index 8cd79ab..1ccd233 100644
--- a/drivers/dax/bus.h
+++ b/drivers/dax/bus.h
@@ -9,7 +9,6 @@ struct dev_dax;
struct resource;
struct dax_device;
struct dax_region;
-void dax_region_put(struct dax_region *dax_region);
/* dax bus specific ioresource flags */
#define IORESOURCE_DAX_STATIC BIT(0)
@@ -49,13 +48,6 @@ void dax_driver_unregister(struct dax_device_driver *dax_drv);
void kill_dev_dax(struct dev_dax *dev_dax);
bool static_dev_dax(struct dev_dax *dev_dax);
-/*
- * While run_dax() is potentially a generic operation that could be
- * defined in include/linux/dax.h we don't want to grow any users
- * outside of drivers/dax/
- */
-void run_dax(struct dax_device *dax_dev);
-
#define MODULE_ALIAS_DAX_DEVICE(type) \
MODULE_ALIAS("dax:t" __stringify(type) "*")
#define DAX_DEVICE_MODALIAS_FMT "dax:t%d"
diff --git a/drivers/dax/cxl.c b/drivers/dax/cxl.c
index ccdf8de..8bc9d04 100644
--- a/drivers/dax/cxl.c
+++ b/drivers/dax/cxl.c
@@ -13,7 +13,6 @@ static int cxl_dax_region_probe(struct device *dev)
struct cxl_region *cxlr = cxlr_dax->cxlr;
struct dax_region *dax_region;
struct dev_dax_data data;
- struct dev_dax *dev_dax;
if (nid == NUMA_NO_NODE)
nid = memory_add_physaddr_to_nid(cxlr_dax->hpa_range.start);
@@ -28,13 +27,8 @@ static int cxl_dax_region_probe(struct device *dev)
.id = -1,
.size = range_len(&cxlr_dax->hpa_range),
};
- dev_dax = devm_create_dev_dax(&data);
- if (IS_ERR(dev_dax))
- return PTR_ERR(dev_dax);
- /* child dev_dax instances now own the lifetime of the dax_region */
- dax_region_put(dax_region);
- return 0;
+ return PTR_ERR_OR_ZERO(devm_create_dev_dax(&data));
}
static struct cxl_driver cxl_dax_region_driver = {
diff --git a/drivers/dax/dax-private.h b/drivers/dax/dax-private.h
index 1c974b7..27cf2daa 100644
--- a/drivers/dax/dax-private.h
+++ b/drivers/dax/dax-private.h
@@ -52,7 +52,8 @@ struct dax_mapping {
* @region - parent region
* @dax_dev - core dax functionality
* @target_node: effective numa node if dev_dax memory range is onlined
- * @id: ida allocated id
+ * @dyn_id: is this a dynamic or statically created instance
+ * @id: ida allocated id when the dax_region is not static
* @ida: mapping id allocator
* @dev - device core
* @pgmap - pgmap for memmap setup / lifetime (driver owned)
@@ -64,6 +65,7 @@ struct dev_dax {
struct dax_device *dax_dev;
unsigned int align;
int target_node;
+ bool dyn_id;
int id;
struct ida ida;
struct device dev;
@@ -76,6 +78,13 @@ struct dev_dax {
} *ranges;
};
+/*
+ * While run_dax() is potentially a generic operation that could be
+ * defined in include/linux/dax.h we don't want to grow any users
+ * outside of drivers/dax/
+ */
+void run_dax(struct dax_device *dax_dev);
+
static inline struct dev_dax *to_dev_dax(struct device *dev)
{
return container_of(dev, struct dev_dax, dev);
diff --git a/drivers/dax/device.c b/drivers/dax/device.c
index af9930c..30665a3 100644
--- a/drivers/dax/device.c
+++ b/drivers/dax/device.c
@@ -396,7 +396,7 @@ static void dev_dax_kill(void *dev_dax)
kill_dev_dax(dev_dax);
}
-int dev_dax_probe(struct dev_dax *dev_dax)
+static int dev_dax_probe(struct dev_dax *dev_dax)
{
struct dax_device *dax_dev = dev_dax->dax_dev;
struct device *dev = &dev_dax->dev;
@@ -471,7 +471,6 @@ int dev_dax_probe(struct dev_dax *dev_dax)
run_dax(dax_dev);
return devm_add_action_or_reset(dev, dev_dax_kill, dev_dax);
}
-EXPORT_SYMBOL_GPL(dev_dax_probe);
static struct dax_device_driver device_dax_driver = {
.probe = dev_dax_probe,
diff --git a/drivers/dax/hmem/hmem.c b/drivers/dax/hmem/hmem.c
index e5fe8b39..5d2ddef 100644
--- a/drivers/dax/hmem/hmem.c
+++ b/drivers/dax/hmem/hmem.c
@@ -16,7 +16,6 @@ static int dax_hmem_probe(struct platform_device *pdev)
struct dax_region *dax_region;
struct memregion_info *mri;
struct dev_dax_data data;
- struct dev_dax *dev_dax;
/*
* @region_idle == true indicates that an administrative agent
@@ -38,13 +37,8 @@ static int dax_hmem_probe(struct platform_device *pdev)
.id = -1,
.size = region_idle ? 0 : range_len(&mri->range),
};
- dev_dax = devm_create_dev_dax(&data);
- if (IS_ERR(dev_dax))
- return PTR_ERR(dev_dax);
- /* child dev_dax instances now own the lifetime of the dax_region */
- dax_region_put(dax_region);
- return 0;
+ return PTR_ERR_OR_ZERO(devm_create_dev_dax(&data));
}
static struct platform_driver dax_hmem_driver = {
diff --git a/drivers/dax/kmem.c b/drivers/dax/kmem.c
index 7b36db6..898ca95 100644
--- a/drivers/dax/kmem.c
+++ b/drivers/dax/kmem.c
@@ -99,7 +99,7 @@ static int dev_dax_kmem_probe(struct dev_dax *dev_dax)
if (!data->res_name)
goto err_res_name;
- rc = memory_group_register_static(numa_node, total_len);
+ rc = memory_group_register_static(numa_node, PFN_UP(total_len));
if (rc < 0)
goto err_reg_mgid;
data->mgid = rc;
diff --git a/drivers/dax/pmem.c b/drivers/dax/pmem.c
index f050ea7..ae0cb11 100644
--- a/drivers/dax/pmem.c
+++ b/drivers/dax/pmem.c
@@ -13,7 +13,6 @@ static struct dev_dax *__dax_pmem_probe(struct device *dev)
int rc, id, region_id;
resource_size_t offset;
struct nd_pfn_sb *pfn_sb;
- struct dev_dax *dev_dax;
struct dev_dax_data data;
struct nd_namespace_io *nsio;
struct dax_region *dax_region;
@@ -65,12 +64,8 @@ static struct dev_dax *__dax_pmem_probe(struct device *dev)
.pgmap = &pgmap,
.size = range_len(&range),
};
- dev_dax = devm_create_dev_dax(&data);
- /* child dev_dax instances now own the lifetime of the dax_region */
- dax_region_put(dax_region);
-
- return dev_dax;
+ return devm_create_dev_dax(&data);
}
static int dax_pmem_probe(struct device *dev)
diff --git a/drivers/dax/super.c b/drivers/dax/super.c
index c4c4728..0da9232 100644
--- a/drivers/dax/super.c
+++ b/drivers/dax/super.c
@@ -203,6 +203,8 @@ size_t dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,
int dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff,
size_t nr_pages)
{
+ int ret;
+
if (!dax_alive(dax_dev))
return -ENXIO;
/*
@@ -213,7 +215,8 @@ int dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff,
if (nr_pages != 1)
return -EIO;
- return dax_dev->ops->zero_page_range(dax_dev, pgoff, nr_pages);
+ ret = dax_dev->ops->zero_page_range(dax_dev, pgoff, nr_pages);
+ return dax_mem2blk_err(ret);
}
EXPORT_SYMBOL_GPL(dax_zero_page_range);
diff --git a/drivers/nvdimm/bus.c b/drivers/nvdimm/bus.c
index 954dbc1..5852fe2 100644
--- a/drivers/nvdimm/bus.c
+++ b/drivers/nvdimm/bus.c
@@ -25,7 +25,7 @@
int nvdimm_major;
static int nvdimm_bus_major;
-struct class *nd_class;
+static struct class *nd_class;
static DEFINE_IDA(nd_ida);
static int to_nd_device_type(const struct device *dev)
diff --git a/drivers/nvdimm/dimm_devs.c b/drivers/nvdimm/dimm_devs.c
index 957f7c3..1273873 100644
--- a/drivers/nvdimm/dimm_devs.c
+++ b/drivers/nvdimm/dimm_devs.c
@@ -349,8 +349,8 @@ static ssize_t available_slots_show(struct device *dev,
}
static DEVICE_ATTR_RO(available_slots);
-ssize_t security_show(struct device *dev,
- struct device_attribute *attr, char *buf)
+static ssize_t security_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
{
struct nvdimm *nvdimm = to_nvdimm(dev);
diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c
index ceea55f..46e094e 100644
--- a/drivers/nvdimm/pmem.c
+++ b/drivers/nvdimm/pmem.c
@@ -260,7 +260,7 @@ __weak long __pmem_direct_access(struct pmem_device *pmem, pgoff_t pgoff,
long actual_nr;
if (mode != DAX_RECOVERY_WRITE)
- return -EIO;
+ return -EHWPOISON;
/*
* Set the recovery stride is set to kernel page size because
diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig
index 4c07d71..f4572a5 100644
--- a/drivers/perf/Kconfig
+++ b/drivers/perf/Kconfig
@@ -221,4 +221,17 @@
source "drivers/perf/amlogic/Kconfig"
+config CXL_PMU
+ tristate "CXL Performance Monitoring Unit"
+ depends on CXL_BUS
+ help
+ Support performance monitoring as defined in CXL rev 3.0
+ section 13.2: Performance Monitoring. CXL components may have
+ one or more CXL Performance Monitoring Units (CPMUs).
+
+ Say 'y/m' to enable a driver that will attach to performance
+ monitoring units and provide standard perf based interfaces.
+
+ If unsure say 'm'.
+
endmenu
diff --git a/drivers/perf/Makefile b/drivers/perf/Makefile
index 5cfe895..16b3ec4 100644
--- a/drivers/perf/Makefile
+++ b/drivers/perf/Makefile
@@ -25,3 +25,4 @@
obj-$(CONFIG_ALIBABA_UNCORE_DRW_PMU) += alibaba_uncore_drw_pmu.o
obj-$(CONFIG_ARM_CORESIGHT_PMU_ARCH_SYSTEM_PMU) += arm_cspmu/
obj-$(CONFIG_MESON_DDR_PMU) += amlogic/
+obj-$(CONFIG_CXL_PMU) += cxl_pmu.o
diff --git a/drivers/perf/cxl_pmu.c b/drivers/perf/cxl_pmu.c
new file mode 100644
index 0000000..0a8f597
--- /dev/null
+++ b/drivers/perf/cxl_pmu.c
@@ -0,0 +1,990 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+/*
+ * Copyright(c) 2023 Huawei
+ *
+ * The CXL 3.0 specification includes a standard Performance Monitoring Unit,
+ * called the CXL PMU, or CPMU. In order to allow a high degree of
+ * implementation flexibility the specification provides a wide range of
+ * options all of which are self describing.
+ *
+ * Details in CXL rev 3.0 section 8.2.7 CPMU Register Interface
+ */
+
+#include <linux/io-64-nonatomic-lo-hi.h>
+#include <linux/perf_event.h>
+#include <linux/bitops.h>
+#include <linux/device.h>
+#include <linux/bits.h>
+#include <linux/list.h>
+#include <linux/bug.h>
+#include <linux/pci.h>
+
+#include "../cxl/cxlpci.h"
+#include "../cxl/cxl.h"
+#include "../cxl/pmu.h"
+
+#define CXL_PMU_CAP_REG 0x0
+#define CXL_PMU_CAP_NUM_COUNTERS_MSK GENMASK_ULL(4, 0)
+#define CXL_PMU_CAP_COUNTER_WIDTH_MSK GENMASK_ULL(15, 8)
+#define CXL_PMU_CAP_NUM_EVN_CAP_REG_SUP_MSK GENMASK_ULL(24, 20)
+#define CXL_PMU_CAP_FILTERS_SUP_MSK GENMASK_ULL(39, 32)
+#define CXL_PMU_FILTER_HDM BIT(0)
+#define CXL_PMU_FILTER_CHAN_RANK_BANK BIT(1)
+#define CXL_PMU_CAP_MSI_N_MSK GENMASK_ULL(47, 44)
+#define CXL_PMU_CAP_WRITEABLE_WHEN_FROZEN BIT_ULL(48)
+#define CXL_PMU_CAP_FREEZE BIT_ULL(49)
+#define CXL_PMU_CAP_INT BIT_ULL(50)
+#define CXL_PMU_CAP_VERSION_MSK GENMASK_ULL(63, 60)
+
+#define CXL_PMU_OVERFLOW_REG 0x10
+#define CXL_PMU_FREEZE_REG 0x18
+#define CXL_PMU_EVENT_CAP_REG(n) (0x100 + 8 * (n))
+#define CXL_PMU_EVENT_CAP_SUPPORTED_EVENTS_MSK GENMASK_ULL(31, 0)
+#define CXL_PMU_EVENT_CAP_GROUP_ID_MSK GENMASK_ULL(47, 32)
+#define CXL_PMU_EVENT_CAP_VENDOR_ID_MSK GENMASK_ULL(63, 48)
+
+#define CXL_PMU_COUNTER_CFG_REG(n) (0x200 + 8 * (n))
+#define CXL_PMU_COUNTER_CFG_TYPE_MSK GENMASK_ULL(1, 0)
+#define CXL_PMU_COUNTER_CFG_TYPE_FREE_RUN 0
+#define CXL_PMU_COUNTER_CFG_TYPE_FIXED_FUN 1
+#define CXL_PMU_COUNTER_CFG_TYPE_CONFIGURABLE 2
+#define CXL_PMU_COUNTER_CFG_ENABLE BIT_ULL(8)
+#define CXL_PMU_COUNTER_CFG_INT_ON_OVRFLW BIT_ULL(9)
+#define CXL_PMU_COUNTER_CFG_FREEZE_ON_OVRFLW BIT_ULL(10)
+#define CXL_PMU_COUNTER_CFG_EDGE BIT_ULL(11)
+#define CXL_PMU_COUNTER_CFG_INVERT BIT_ULL(12)
+#define CXL_PMU_COUNTER_CFG_THRESHOLD_MSK GENMASK_ULL(23, 16)
+#define CXL_PMU_COUNTER_CFG_EVENTS_MSK GENMASK_ULL(55, 24)
+#define CXL_PMU_COUNTER_CFG_EVENT_GRP_ID_IDX_MSK GENMASK_ULL(63, 59)
+
+#define CXL_PMU_FILTER_CFG_REG(n, f) (0x400 + 4 * ((f) + (n) * 8))
+#define CXL_PMU_FILTER_CFG_VALUE_MSK GENMASK(15, 0)
+
+#define CXL_PMU_COUNTER_REG(n) (0xc00 + 8 * (n))
+
+/* CXL rev 3.0 Table 13-5 Events under CXL Vendor ID */
+#define CXL_PMU_GID_CLOCK_TICKS 0x00
+#define CXL_PMU_GID_D2H_REQ 0x0010
+#define CXL_PMU_GID_D2H_RSP 0x0011
+#define CXL_PMU_GID_H2D_REQ 0x0012
+#define CXL_PMU_GID_H2D_RSP 0x0013
+#define CXL_PMU_GID_CACHE_DATA 0x0014
+#define CXL_PMU_GID_M2S_REQ 0x0020
+#define CXL_PMU_GID_M2S_RWD 0x0021
+#define CXL_PMU_GID_M2S_BIRSP 0x0022
+#define CXL_PMU_GID_S2M_BISNP 0x0023
+#define CXL_PMU_GID_S2M_NDR 0x0024
+#define CXL_PMU_GID_S2M_DRS 0x0025
+#define CXL_PMU_GID_DDR 0x8000
+
+static int cxl_pmu_cpuhp_state_num;
+
+struct cxl_pmu_ev_cap {
+ u16 vid;
+ u16 gid;
+ u32 msk;
+ union {
+ int counter_idx; /* fixed counters */
+ int event_idx; /* configurable counters */
+ };
+ struct list_head node;
+};
+
+#define CXL_PMU_MAX_COUNTERS 64
+struct cxl_pmu_info {
+ struct pmu pmu;
+ void __iomem *base;
+ struct perf_event **hw_events;
+ struct list_head event_caps_configurable;
+ struct list_head event_caps_fixed;
+ DECLARE_BITMAP(used_counter_bm, CXL_PMU_MAX_COUNTERS);
+ DECLARE_BITMAP(conf_counter_bm, CXL_PMU_MAX_COUNTERS);
+ u16 counter_width;
+ u8 num_counters;
+ u8 num_event_capabilities;
+ int on_cpu;
+ struct hlist_node node;
+ bool filter_hdm;
+ int irq;
+};
+
+#define pmu_to_cxl_pmu_info(_pmu) container_of(_pmu, struct cxl_pmu_info, pmu)
+
+/*
+ * All CPMU counters are discoverable via the Event Capabilities Registers.
+ * Each Event Capability register contains a a VID / GroupID.
+ * A counter may then count any combination (by summing) of events in
+ * that group which are in the Supported Events Bitmask.
+ * However, there are some complexities to the scheme.
+ * - Fixed function counters refer to an Event Capabilities register.
+ * That event capability register is not then used for Configurable
+ * counters.
+ */
+static int cxl_pmu_parse_caps(struct device *dev, struct cxl_pmu_info *info)
+{
+ unsigned long fixed_counter_event_cap_bm = 0;
+ void __iomem *base = info->base;
+ bool freeze_for_enable;
+ u64 val, eval;
+ int i;
+
+ val = readq(base + CXL_PMU_CAP_REG);
+ freeze_for_enable = FIELD_GET(CXL_PMU_CAP_WRITEABLE_WHEN_FROZEN, val) &&
+ FIELD_GET(CXL_PMU_CAP_FREEZE, val);
+ if (!freeze_for_enable) {
+ dev_err(dev, "Counters not writable while frozen\n");
+ return -ENODEV;
+ }
+
+ info->num_counters = FIELD_GET(CXL_PMU_CAP_NUM_COUNTERS_MSK, val) + 1;
+ info->counter_width = FIELD_GET(CXL_PMU_CAP_COUNTER_WIDTH_MSK, val);
+ info->num_event_capabilities = FIELD_GET(CXL_PMU_CAP_NUM_EVN_CAP_REG_SUP_MSK, val) + 1;
+
+ info->filter_hdm = FIELD_GET(CXL_PMU_CAP_FILTERS_SUP_MSK, val) & CXL_PMU_FILTER_HDM;
+ if (FIELD_GET(CXL_PMU_CAP_INT, val))
+ info->irq = FIELD_GET(CXL_PMU_CAP_MSI_N_MSK, val);
+ else
+ info->irq = -1;
+
+ /* First handle fixed function counters; note if configurable counters found */
+ for (i = 0; i < info->num_counters; i++) {
+ struct cxl_pmu_ev_cap *pmu_ev;
+ u32 events_msk;
+ u8 group_idx;
+
+ val = readq(base + CXL_PMU_COUNTER_CFG_REG(i));
+
+ if (FIELD_GET(CXL_PMU_COUNTER_CFG_TYPE_MSK, val) ==
+ CXL_PMU_COUNTER_CFG_TYPE_CONFIGURABLE) {
+ set_bit(i, info->conf_counter_bm);
+ }
+
+ if (FIELD_GET(CXL_PMU_COUNTER_CFG_TYPE_MSK, val) !=
+ CXL_PMU_COUNTER_CFG_TYPE_FIXED_FUN)
+ continue;
+
+ /* In this case we know which fields are const */
+ group_idx = FIELD_GET(CXL_PMU_COUNTER_CFG_EVENT_GRP_ID_IDX_MSK, val);
+ events_msk = FIELD_GET(CXL_PMU_COUNTER_CFG_EVENTS_MSK, val);
+ eval = readq(base + CXL_PMU_EVENT_CAP_REG(group_idx));
+ pmu_ev = devm_kzalloc(dev, sizeof(*pmu_ev), GFP_KERNEL);
+ if (!pmu_ev)
+ return -ENOMEM;
+
+ pmu_ev->vid = FIELD_GET(CXL_PMU_EVENT_CAP_VENDOR_ID_MSK, eval);
+ pmu_ev->gid = FIELD_GET(CXL_PMU_EVENT_CAP_GROUP_ID_MSK, eval);
+ /* For a fixed purpose counter use the events mask from the counter CFG */
+ pmu_ev->msk = events_msk;
+ pmu_ev->counter_idx = i;
+ /* This list add is never unwound as all entries deleted on remove */
+ list_add(&pmu_ev->node, &info->event_caps_fixed);
+ /*
+ * Configurable counters must not use an Event Capability registers that
+ * is in use for a Fixed counter
+ */
+ set_bit(group_idx, &fixed_counter_event_cap_bm);
+ }
+
+ if (!bitmap_empty(info->conf_counter_bm, CXL_PMU_MAX_COUNTERS)) {
+ struct cxl_pmu_ev_cap *pmu_ev;
+ int j;
+ /* Walk event capabilities unused by fixed counters */
+ for_each_clear_bit(j, &fixed_counter_event_cap_bm,
+ info->num_event_capabilities) {
+ pmu_ev = devm_kzalloc(dev, sizeof(*pmu_ev), GFP_KERNEL);
+ if (!pmu_ev)
+ return -ENOMEM;
+
+ eval = readq(base + CXL_PMU_EVENT_CAP_REG(j));
+ pmu_ev->vid = FIELD_GET(CXL_PMU_EVENT_CAP_VENDOR_ID_MSK, eval);
+ pmu_ev->gid = FIELD_GET(CXL_PMU_EVENT_CAP_GROUP_ID_MSK, eval);
+ pmu_ev->msk = FIELD_GET(CXL_PMU_EVENT_CAP_SUPPORTED_EVENTS_MSK, eval);
+ pmu_ev->event_idx = j;
+ list_add(&pmu_ev->node, &info->event_caps_configurable);
+ }
+ }
+
+ return 0;
+}
+
+static ssize_t cxl_pmu_format_sysfs_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct dev_ext_attribute *eattr;
+
+ eattr = container_of(attr, struct dev_ext_attribute, attr);
+
+ return sysfs_emit(buf, "%s\n", (char *)eattr->var);
+}
+
+#define CXL_PMU_FORMAT_ATTR(_name, _format)\
+ (&((struct dev_ext_attribute[]) { \
+ { \
+ .attr = __ATTR(_name, 0444, \
+ cxl_pmu_format_sysfs_show, NULL), \
+ .var = (void *)_format \
+ } \
+ })[0].attr.attr)
+
+enum {
+ cxl_pmu_mask_attr,
+ cxl_pmu_gid_attr,
+ cxl_pmu_vid_attr,
+ cxl_pmu_threshold_attr,
+ cxl_pmu_invert_attr,
+ cxl_pmu_edge_attr,
+ cxl_pmu_hdm_filter_en_attr,
+ cxl_pmu_hdm_attr,
+};
+
+static struct attribute *cxl_pmu_format_attr[] = {
+ [cxl_pmu_mask_attr] = CXL_PMU_FORMAT_ATTR(mask, "config:0-31"),
+ [cxl_pmu_gid_attr] = CXL_PMU_FORMAT_ATTR(gid, "config:32-47"),
+ [cxl_pmu_vid_attr] = CXL_PMU_FORMAT_ATTR(vid, "config:48-63"),
+ [cxl_pmu_threshold_attr] = CXL_PMU_FORMAT_ATTR(threshold, "config1:0-15"),
+ [cxl_pmu_invert_attr] = CXL_PMU_FORMAT_ATTR(invert, "config1:16"),
+ [cxl_pmu_edge_attr] = CXL_PMU_FORMAT_ATTR(edge, "config1:17"),
+ [cxl_pmu_hdm_filter_en_attr] = CXL_PMU_FORMAT_ATTR(hdm_filter_en, "config1:18"),
+ [cxl_pmu_hdm_attr] = CXL_PMU_FORMAT_ATTR(hdm, "config2:0-15"),
+ NULL
+};
+
+#define CXL_PMU_ATTR_CONFIG_MASK_MSK GENMASK_ULL(31, 0)
+#define CXL_PMU_ATTR_CONFIG_GID_MSK GENMASK_ULL(47, 32)
+#define CXL_PMU_ATTR_CONFIG_VID_MSK GENMASK_ULL(63, 48)
+#define CXL_PMU_ATTR_CONFIG1_THRESHOLD_MSK GENMASK_ULL(15, 0)
+#define CXL_PMU_ATTR_CONFIG1_INVERT_MSK BIT(16)
+#define CXL_PMU_ATTR_CONFIG1_EDGE_MSK BIT(17)
+#define CXL_PMU_ATTR_CONFIG1_FILTER_EN_MSK BIT(18)
+#define CXL_PMU_ATTR_CONFIG2_HDM_MSK GENMASK(15, 0)
+
+static umode_t cxl_pmu_format_is_visible(struct kobject *kobj,
+ struct attribute *attr, int a)
+{
+ struct device *dev = kobj_to_dev(kobj);
+ struct cxl_pmu_info *info = dev_get_drvdata(dev);
+
+ /*
+ * Filter capability at the CPMU level, so hide the attributes if the particular
+ * filter is not supported.
+ */
+ if (!info->filter_hdm &&
+ (attr == cxl_pmu_format_attr[cxl_pmu_hdm_filter_en_attr] ||
+ attr == cxl_pmu_format_attr[cxl_pmu_hdm_attr]))
+ return 0;
+
+ return attr->mode;
+}
+
+static const struct attribute_group cxl_pmu_format_group = {
+ .name = "format",
+ .attrs = cxl_pmu_format_attr,
+ .is_visible = cxl_pmu_format_is_visible,
+};
+
+static u32 cxl_pmu_config_get_mask(struct perf_event *event)
+{
+ return FIELD_GET(CXL_PMU_ATTR_CONFIG_MASK_MSK, event->attr.config);
+}
+
+static u16 cxl_pmu_config_get_gid(struct perf_event *event)
+{
+ return FIELD_GET(CXL_PMU_ATTR_CONFIG_GID_MSK, event->attr.config);
+}
+
+static u16 cxl_pmu_config_get_vid(struct perf_event *event)
+{
+ return FIELD_GET(CXL_PMU_ATTR_CONFIG_VID_MSK, event->attr.config);
+}
+
+static u8 cxl_pmu_config1_get_threshold(struct perf_event *event)
+{
+ return FIELD_GET(CXL_PMU_ATTR_CONFIG1_THRESHOLD_MSK, event->attr.config1);
+}
+
+static bool cxl_pmu_config1_get_invert(struct perf_event *event)
+{
+ return FIELD_GET(CXL_PMU_ATTR_CONFIG1_INVERT_MSK, event->attr.config1);
+}
+
+static bool cxl_pmu_config1_get_edge(struct perf_event *event)
+{
+ return FIELD_GET(CXL_PMU_ATTR_CONFIG1_EDGE_MSK, event->attr.config1);
+}
+
+/*
+ * CPMU specification allows for 8 filters, each with a 16 bit value...
+ * So we need to find 8x16bits to store it in.
+ * As the value used for disable is 0xffff, a separate enable switch
+ * is needed.
+ */
+
+static bool cxl_pmu_config1_hdm_filter_en(struct perf_event *event)
+{
+ return FIELD_GET(CXL_PMU_ATTR_CONFIG1_FILTER_EN_MSK, event->attr.config1);
+}
+
+static u16 cxl_pmu_config2_get_hdm_decoder(struct perf_event *event)
+{
+ return FIELD_GET(CXL_PMU_ATTR_CONFIG2_HDM_MSK, event->attr.config2);
+}
+
+static ssize_t cxl_pmu_event_sysfs_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct perf_pmu_events_attr *pmu_attr =
+ container_of(attr, struct perf_pmu_events_attr, attr);
+
+ return sysfs_emit(buf, "config=%#llx\n", pmu_attr->id);
+}
+
+#define CXL_PMU_EVENT_ATTR(_name, _vid, _gid, _msk) \
+ PMU_EVENT_ATTR_ID(_name, cxl_pmu_event_sysfs_show, \
+ ((u64)(_vid) << 48) | ((u64)(_gid) << 32) | (u64)(_msk))
+
+/* For CXL spec defined events */
+#define CXL_PMU_EVENT_CXL_ATTR(_name, _gid, _msk) \
+ CXL_PMU_EVENT_ATTR(_name, PCI_DVSEC_VENDOR_ID_CXL, _gid, _msk)
+
+static struct attribute *cxl_pmu_event_attrs[] = {
+ CXL_PMU_EVENT_CXL_ATTR(clock_ticks, CXL_PMU_GID_CLOCK_TICKS, BIT(0)),
+ /* CXL rev 3.0 Table 3-17 - Device to Host Requests */
+ CXL_PMU_EVENT_CXL_ATTR(d2h_req_rdcurr, CXL_PMU_GID_D2H_REQ, BIT(1)),
+ CXL_PMU_EVENT_CXL_ATTR(d2h_req_rdown, CXL_PMU_GID_D2H_REQ, BIT(2)),
+ CXL_PMU_EVENT_CXL_ATTR(d2h_req_rdshared, CXL_PMU_GID_D2H_REQ, BIT(3)),
+ CXL_PMU_EVENT_CXL_ATTR(d2h_req_rdany, CXL_PMU_GID_D2H_REQ, BIT(4)),
+ CXL_PMU_EVENT_CXL_ATTR(d2h_req_rdownnodata, CXL_PMU_GID_D2H_REQ, BIT(5)),
+ CXL_PMU_EVENT_CXL_ATTR(d2h_req_itomwr, CXL_PMU_GID_D2H_REQ, BIT(6)),
+ CXL_PMU_EVENT_CXL_ATTR(d2h_req_wrcurr, CXL_PMU_GID_D2H_REQ, BIT(7)),
+ CXL_PMU_EVENT_CXL_ATTR(d2h_req_clflush, CXL_PMU_GID_D2H_REQ, BIT(8)),
+ CXL_PMU_EVENT_CXL_ATTR(d2h_req_cleanevict, CXL_PMU_GID_D2H_REQ, BIT(9)),
+ CXL_PMU_EVENT_CXL_ATTR(d2h_req_dirtyevict, CXL_PMU_GID_D2H_REQ, BIT(10)),
+ CXL_PMU_EVENT_CXL_ATTR(d2h_req_cleanevictnodata, CXL_PMU_GID_D2H_REQ, BIT(11)),
+ CXL_PMU_EVENT_CXL_ATTR(d2h_req_wowrinv, CXL_PMU_GID_D2H_REQ, BIT(12)),
+ CXL_PMU_EVENT_CXL_ATTR(d2h_req_wowrinvf, CXL_PMU_GID_D2H_REQ, BIT(13)),
+ CXL_PMU_EVENT_CXL_ATTR(d2h_req_wrinv, CXL_PMU_GID_D2H_REQ, BIT(14)),
+ CXL_PMU_EVENT_CXL_ATTR(d2h_req_cacheflushed, CXL_PMU_GID_D2H_REQ, BIT(16)),
+ /* CXL rev 3.0 Table 3-20 - D2H Repsonse Encodings */
+ CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspihiti, CXL_PMU_GID_D2H_RSP, BIT(4)),
+ CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspvhitv, CXL_PMU_GID_D2H_RSP, BIT(6)),
+ CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspihitse, CXL_PMU_GID_D2H_RSP, BIT(5)),
+ CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspshitse, CXL_PMU_GID_D2H_RSP, BIT(1)),
+ CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspsfwdm, CXL_PMU_GID_D2H_RSP, BIT(7)),
+ CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspifwdm, CXL_PMU_GID_D2H_RSP, BIT(15)),
+ CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspvfwdv, CXL_PMU_GID_D2H_RSP, BIT(22)),
+ /* CXL rev 3.0 Table 3-21 - CXL.cache - Mapping of H2D Requests to D2H Responses */
+ CXL_PMU_EVENT_CXL_ATTR(h2d_req_snpdata, CXL_PMU_GID_H2D_REQ, BIT(1)),
+ CXL_PMU_EVENT_CXL_ATTR(h2d_req_snpinv, CXL_PMU_GID_H2D_REQ, BIT(2)),
+ CXL_PMU_EVENT_CXL_ATTR(h2d_req_snpcur, CXL_PMU_GID_H2D_REQ, BIT(3)),
+ /* CXL rev 3.0 Table 3-22 - H2D Response Opcode Encodings */
+ CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_writepull, CXL_PMU_GID_H2D_RSP, BIT(1)),
+ CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_go, CXL_PMU_GID_H2D_RSP, BIT(4)),
+ CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_gowritepull, CXL_PMU_GID_H2D_RSP, BIT(5)),
+ CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_extcmp, CXL_PMU_GID_H2D_RSP, BIT(6)),
+ CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_gowritepulldrop, CXL_PMU_GID_H2D_RSP, BIT(8)),
+ CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_fastgowritepull, CXL_PMU_GID_H2D_RSP, BIT(13)),
+ CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_goerrwritepull, CXL_PMU_GID_H2D_RSP, BIT(15)),
+ /* CXL rev 3.0 Table 13-5 directly lists these */
+ CXL_PMU_EVENT_CXL_ATTR(cachedata_d2h_data, CXL_PMU_GID_CACHE_DATA, BIT(0)),
+ CXL_PMU_EVENT_CXL_ATTR(cachedata_h2d_data, CXL_PMU_GID_CACHE_DATA, BIT(1)),
+ /* CXL rev 3.0 Table 3-29 M2S Req Memory Opcodes */
+ CXL_PMU_EVENT_CXL_ATTR(m2s_req_meminv, CXL_PMU_GID_M2S_REQ, BIT(0)),
+ CXL_PMU_EVENT_CXL_ATTR(m2s_req_memrd, CXL_PMU_GID_M2S_REQ, BIT(1)),
+ CXL_PMU_EVENT_CXL_ATTR(m2s_req_memrddata, CXL_PMU_GID_M2S_REQ, BIT(2)),
+ CXL_PMU_EVENT_CXL_ATTR(m2s_req_memrdfwd, CXL_PMU_GID_M2S_REQ, BIT(3)),
+ CXL_PMU_EVENT_CXL_ATTR(m2s_req_memwrfwd, CXL_PMU_GID_M2S_REQ, BIT(4)),
+ CXL_PMU_EVENT_CXL_ATTR(m2s_req_memspecrd, CXL_PMU_GID_M2S_REQ, BIT(8)),
+ CXL_PMU_EVENT_CXL_ATTR(m2s_req_meminvnt, CXL_PMU_GID_M2S_REQ, BIT(9)),
+ CXL_PMU_EVENT_CXL_ATTR(m2s_req_memcleanevict, CXL_PMU_GID_M2S_REQ, BIT(10)),
+ /* CXL rev 3.0 Table 3-35 M2S RwD Memory Opcodes */
+ CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memwr, CXL_PMU_GID_M2S_RWD, BIT(1)),
+ CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memwrptl, CXL_PMU_GID_M2S_RWD, BIT(2)),
+ CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_biconflict, CXL_PMU_GID_M2S_RWD, BIT(4)),
+ /* CXL rev 3.0 Table 3-38 M2S BIRsp Memory Opcodes */
+ CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_i, CXL_PMU_GID_M2S_BIRSP, BIT(0)),
+ CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_s, CXL_PMU_GID_M2S_BIRSP, BIT(1)),
+ CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_e, CXL_PMU_GID_M2S_BIRSP, BIT(2)),
+ CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_iblk, CXL_PMU_GID_M2S_BIRSP, BIT(4)),
+ CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_sblk, CXL_PMU_GID_M2S_BIRSP, BIT(5)),
+ CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_eblk, CXL_PMU_GID_M2S_BIRSP, BIT(6)),
+ /* CXL rev 3.0 Table 3-40 S2M BISnp Opcodes */
+ CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_cur, CXL_PMU_GID_S2M_BISNP, BIT(0)),
+ CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_data, CXL_PMU_GID_S2M_BISNP, BIT(1)),
+ CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_inv, CXL_PMU_GID_S2M_BISNP, BIT(2)),
+ CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_curblk, CXL_PMU_GID_S2M_BISNP, BIT(4)),
+ CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_datblk, CXL_PMU_GID_S2M_BISNP, BIT(5)),
+ CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_invblk, CXL_PMU_GID_S2M_BISNP, BIT(6)),
+ /* CXL rev 3.0 Table 3-43 S2M NDR Opcopdes */
+ CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmp, CXL_PMU_GID_S2M_NDR, BIT(0)),
+ CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmps, CXL_PMU_GID_S2M_NDR, BIT(1)),
+ CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmpe, CXL_PMU_GID_S2M_NDR, BIT(2)),
+ CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_biconflictack, CXL_PMU_GID_S2M_NDR, BIT(3)),
+ /* CXL rev 3.0 Table 3-46 S2M DRS opcodes */
+ CXL_PMU_EVENT_CXL_ATTR(s2m_drs_memdata, CXL_PMU_GID_S2M_DRS, BIT(0)),
+ CXL_PMU_EVENT_CXL_ATTR(s2m_drs_memdatanxm, CXL_PMU_GID_S2M_DRS, BIT(1)),
+ /* CXL rev 3.0 Table 13-5 directly lists these */
+ CXL_PMU_EVENT_CXL_ATTR(ddr_act, CXL_PMU_GID_DDR, BIT(0)),
+ CXL_PMU_EVENT_CXL_ATTR(ddr_pre, CXL_PMU_GID_DDR, BIT(1)),
+ CXL_PMU_EVENT_CXL_ATTR(ddr_casrd, CXL_PMU_GID_DDR, BIT(2)),
+ CXL_PMU_EVENT_CXL_ATTR(ddr_caswr, CXL_PMU_GID_DDR, BIT(3)),
+ CXL_PMU_EVENT_CXL_ATTR(ddr_refresh, CXL_PMU_GID_DDR, BIT(4)),
+ CXL_PMU_EVENT_CXL_ATTR(ddr_selfrefreshent, CXL_PMU_GID_DDR, BIT(5)),
+ CXL_PMU_EVENT_CXL_ATTR(ddr_rfm, CXL_PMU_GID_DDR, BIT(6)),
+ NULL
+};
+
+static struct cxl_pmu_ev_cap *cxl_pmu_find_fixed_counter_ev_cap(struct cxl_pmu_info *info,
+ int vid, int gid, int msk)
+{
+ struct cxl_pmu_ev_cap *pmu_ev;
+
+ list_for_each_entry(pmu_ev, &info->event_caps_fixed, node) {
+ if (vid != pmu_ev->vid || gid != pmu_ev->gid)
+ continue;
+
+ /* Precise match for fixed counter */
+ if (msk == pmu_ev->msk)
+ return pmu_ev;
+ }
+
+ return ERR_PTR(-EINVAL);
+}
+
+static struct cxl_pmu_ev_cap *cxl_pmu_find_config_counter_ev_cap(struct cxl_pmu_info *info,
+ int vid, int gid, int msk)
+{
+ struct cxl_pmu_ev_cap *pmu_ev;
+
+ list_for_each_entry(pmu_ev, &info->event_caps_configurable, node) {
+ if (vid != pmu_ev->vid || gid != pmu_ev->gid)
+ continue;
+
+ /* Request mask must be subset of supported */
+ if (msk & ~pmu_ev->msk)
+ continue;
+
+ return pmu_ev;
+ }
+
+ return ERR_PTR(-EINVAL);
+}
+
+static umode_t cxl_pmu_event_is_visible(struct kobject *kobj, struct attribute *attr, int a)
+{
+ struct device_attribute *dev_attr = container_of(attr, struct device_attribute, attr);
+ struct perf_pmu_events_attr *pmu_attr =
+ container_of(dev_attr, struct perf_pmu_events_attr, attr);
+ struct device *dev = kobj_to_dev(kobj);
+ struct cxl_pmu_info *info = dev_get_drvdata(dev);
+ int vid = FIELD_GET(CXL_PMU_ATTR_CONFIG_VID_MSK, pmu_attr->id);
+ int gid = FIELD_GET(CXL_PMU_ATTR_CONFIG_GID_MSK, pmu_attr->id);
+ int msk = FIELD_GET(CXL_PMU_ATTR_CONFIG_MASK_MSK, pmu_attr->id);
+
+ if (!IS_ERR(cxl_pmu_find_fixed_counter_ev_cap(info, vid, gid, msk)))
+ return attr->mode;
+
+ if (!IS_ERR(cxl_pmu_find_config_counter_ev_cap(info, vid, gid, msk)))
+ return attr->mode;
+
+ return 0;
+}
+
+static const struct attribute_group cxl_pmu_events = {
+ .name = "events",
+ .attrs = cxl_pmu_event_attrs,
+ .is_visible = cxl_pmu_event_is_visible,
+};
+
+static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct cxl_pmu_info *info = dev_get_drvdata(dev);
+
+ return cpumap_print_to_pagebuf(true, buf, cpumask_of(info->on_cpu));
+}
+static DEVICE_ATTR_RO(cpumask);
+
+static struct attribute *cxl_pmu_cpumask_attrs[] = {
+ &dev_attr_cpumask.attr,
+ NULL
+};
+
+static const struct attribute_group cxl_pmu_cpumask_group = {
+ .attrs = cxl_pmu_cpumask_attrs,
+};
+
+static const struct attribute_group *cxl_pmu_attr_groups[] = {
+ &cxl_pmu_events,
+ &cxl_pmu_format_group,
+ &cxl_pmu_cpumask_group,
+ NULL
+};
+
+/* If counter_idx == NULL, don't try to allocate a counter. */
+static int cxl_pmu_get_event_idx(struct perf_event *event, int *counter_idx,
+ int *event_idx)
+{
+ struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu);
+ DECLARE_BITMAP(configurable_and_free, CXL_PMU_MAX_COUNTERS);
+ struct cxl_pmu_ev_cap *pmu_ev;
+ u32 mask;
+ u16 gid, vid;
+ int i;
+
+ vid = cxl_pmu_config_get_vid(event);
+ gid = cxl_pmu_config_get_gid(event);
+ mask = cxl_pmu_config_get_mask(event);
+
+ pmu_ev = cxl_pmu_find_fixed_counter_ev_cap(info, vid, gid, mask);
+ if (!IS_ERR(pmu_ev)) {
+ if (!counter_idx)
+ return 0;
+ if (!test_bit(pmu_ev->counter_idx, info->used_counter_bm)) {
+ *counter_idx = pmu_ev->counter_idx;
+ return 0;
+ }
+ /* Fixed counter is in use, but maybe a configurable one? */
+ }
+
+ pmu_ev = cxl_pmu_find_config_counter_ev_cap(info, vid, gid, mask);
+ if (!IS_ERR(pmu_ev)) {
+ if (!counter_idx)
+ return 0;
+
+ bitmap_andnot(configurable_and_free, info->conf_counter_bm,
+ info->used_counter_bm, CXL_PMU_MAX_COUNTERS);
+
+ i = find_first_bit(configurable_and_free, CXL_PMU_MAX_COUNTERS);
+ if (i == CXL_PMU_MAX_COUNTERS)
+ return -EINVAL;
+
+ *counter_idx = i;
+ return 0;
+ }
+
+ return -EINVAL;
+}
+
+static int cxl_pmu_event_init(struct perf_event *event)
+{
+ struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu);
+ int rc;
+
+ /* Top level type sanity check - is this a Hardware Event being requested */
+ if (event->attr.type != event->pmu->type)
+ return -ENOENT;
+
+ if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
+ return -EOPNOTSUPP;
+ /* TODO: Validation of any filter */
+
+ /*
+ * Verify that it is possible to count what was requested. Either must
+ * be a fixed counter that is a precise match or a configurable counter
+ * where this is a subset.
+ */
+ rc = cxl_pmu_get_event_idx(event, NULL, NULL);
+ if (rc < 0)
+ return rc;
+
+ event->cpu = info->on_cpu;
+
+ return 0;
+}
+
+static void cxl_pmu_enable(struct pmu *pmu)
+{
+ struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(pmu);
+ void __iomem *base = info->base;
+
+ /* Can assume frozen at this stage */
+ writeq(0, base + CXL_PMU_FREEZE_REG);
+}
+
+static void cxl_pmu_disable(struct pmu *pmu)
+{
+ struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(pmu);
+ void __iomem *base = info->base;
+
+ /*
+ * Whilst bits above number of counters are RsvdZ
+ * they are unlikely to be repurposed given
+ * number of counters is allowed to be 64 leaving
+ * no reserved bits. Hence this is only slightly
+ * naughty.
+ */
+ writeq(GENMASK_ULL(63, 0), base + CXL_PMU_FREEZE_REG);
+}
+
+static void cxl_pmu_event_start(struct perf_event *event, int flags)
+{
+ struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu);
+ struct hw_perf_event *hwc = &event->hw;
+ void __iomem *base = info->base;
+ u64 cfg;
+
+ /*
+ * All paths to here should either set these flags directly or
+ * call cxl_pmu_event_stop() which will ensure the correct state.
+ */
+ if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
+ return;
+
+ WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
+ hwc->state = 0;
+
+ /*
+ * Currently only hdm filter control is implemnted, this code will
+ * want generalizing when more filters are added.
+ */
+ if (info->filter_hdm) {
+ if (cxl_pmu_config1_hdm_filter_en(event))
+ cfg = cxl_pmu_config2_get_hdm_decoder(event);
+ else
+ cfg = GENMASK(15, 0); /* No filtering if 0xFFFF_FFFF */
+ writeq(cfg, base + CXL_PMU_FILTER_CFG_REG(hwc->idx, 0));
+ }
+
+ cfg = readq(base + CXL_PMU_COUNTER_CFG_REG(hwc->idx));
+ cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_INT_ON_OVRFLW, 1);
+ cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_FREEZE_ON_OVRFLW, 1);
+ cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_ENABLE, 1);
+ cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_EDGE,
+ cxl_pmu_config1_get_edge(event) ? 1 : 0);
+ cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_INVERT,
+ cxl_pmu_config1_get_invert(event) ? 1 : 0);
+
+ /* Fixed purpose counters have next two fields RO */
+ if (test_bit(hwc->idx, info->conf_counter_bm)) {
+ cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_EVENT_GRP_ID_IDX_MSK,
+ hwc->event_base);
+ cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_EVENTS_MSK,
+ cxl_pmu_config_get_mask(event));
+ }
+ cfg &= ~CXL_PMU_COUNTER_CFG_THRESHOLD_MSK;
+ /*
+ * For events that generate only 1 count per clock the CXL 3.0 spec
+ * states the threshold shall be set to 1 but if set to 0 it will
+ * count the raw value anwyay?
+ * There is no definition of what events will count multiple per cycle
+ * and hence to which non 1 values of threshold can apply.
+ * (CXL 3.0 8.2.7.2.1 Counter Configuration - threshold field definition)
+ */
+ cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_THRESHOLD_MSK,
+ cxl_pmu_config1_get_threshold(event));
+ writeq(cfg, base + CXL_PMU_COUNTER_CFG_REG(hwc->idx));
+
+ local64_set(&hwc->prev_count, 0);
+ writeq(0, base + CXL_PMU_COUNTER_REG(hwc->idx));
+
+ perf_event_update_userpage(event);
+}
+
+static u64 cxl_pmu_read_counter(struct perf_event *event)
+{
+ struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu);
+ void __iomem *base = info->base;
+
+ return readq(base + CXL_PMU_COUNTER_REG(event->hw.idx));
+}
+
+static void __cxl_pmu_read(struct perf_event *event, bool overflow)
+{
+ struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu);
+ struct hw_perf_event *hwc = &event->hw;
+ u64 new_cnt, prev_cnt, delta;
+
+ do {
+ prev_cnt = local64_read(&hwc->prev_count);
+ new_cnt = cxl_pmu_read_counter(event);
+ } while (local64_cmpxchg(&hwc->prev_count, prev_cnt, new_cnt) != prev_cnt);
+
+ /*
+ * If we know an overflow occur then take that into account.
+ * Note counter is not reset as that would lose events
+ */
+ delta = (new_cnt - prev_cnt) & GENMASK_ULL(info->counter_width - 1, 0);
+ if (overflow && delta < GENMASK_ULL(info->counter_width - 1, 0))
+ delta += (1UL << info->counter_width);
+
+ local64_add(delta, &event->count);
+}
+
+static void cxl_pmu_read(struct perf_event *event)
+{
+ __cxl_pmu_read(event, false);
+}
+
+static void cxl_pmu_event_stop(struct perf_event *event, int flags)
+{
+ struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu);
+ void __iomem *base = info->base;
+ struct hw_perf_event *hwc = &event->hw;
+ u64 cfg;
+
+ cxl_pmu_read(event);
+ WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
+ hwc->state |= PERF_HES_STOPPED;
+
+ cfg = readq(base + CXL_PMU_COUNTER_CFG_REG(hwc->idx));
+ cfg &= ~(FIELD_PREP(CXL_PMU_COUNTER_CFG_INT_ON_OVRFLW, 1) |
+ FIELD_PREP(CXL_PMU_COUNTER_CFG_ENABLE, 1));
+ writeq(cfg, base + CXL_PMU_COUNTER_CFG_REG(hwc->idx));
+
+ hwc->state |= PERF_HES_UPTODATE;
+}
+
+static int cxl_pmu_event_add(struct perf_event *event, int flags)
+{
+ struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu);
+ struct hw_perf_event *hwc = &event->hw;
+ int idx, rc;
+ int event_idx = 0;
+
+ hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
+
+ rc = cxl_pmu_get_event_idx(event, &idx, &event_idx);
+ if (rc < 0)
+ return rc;
+
+ hwc->idx = idx;
+
+ /* Only set for configurable counters */
+ hwc->event_base = event_idx;
+ info->hw_events[idx] = event;
+ set_bit(idx, info->used_counter_bm);
+
+ if (flags & PERF_EF_START)
+ cxl_pmu_event_start(event, PERF_EF_RELOAD);
+
+ return 0;
+}
+
+static void cxl_pmu_event_del(struct perf_event *event, int flags)
+{
+ struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu);
+ struct hw_perf_event *hwc = &event->hw;
+
+ cxl_pmu_event_stop(event, PERF_EF_UPDATE);
+ clear_bit(hwc->idx, info->used_counter_bm);
+ info->hw_events[hwc->idx] = NULL;
+ perf_event_update_userpage(event);
+}
+
+static irqreturn_t cxl_pmu_irq(int irq, void *data)
+{
+ struct cxl_pmu_info *info = data;
+ void __iomem *base = info->base;
+ u64 overflowed;
+ DECLARE_BITMAP(overflowedbm, 64);
+ int i;
+
+ overflowed = readq(base + CXL_PMU_OVERFLOW_REG);
+
+ /* Interrupt may be shared, so maybe it isn't ours */
+ if (!overflowed)
+ return IRQ_NONE;
+
+ bitmap_from_arr64(overflowedbm, &overflowed, 64);
+ for_each_set_bit(i, overflowedbm, info->num_counters) {
+ struct perf_event *event = info->hw_events[i];
+
+ if (!event) {
+ dev_dbg(info->pmu.dev,
+ "overflow but on non enabled counter %d\n", i);
+ continue;
+ }
+
+ __cxl_pmu_read(event, true);
+ }
+
+ writeq(overflowed, base + CXL_PMU_OVERFLOW_REG);
+
+ return IRQ_HANDLED;
+}
+
+static void cxl_pmu_perf_unregister(void *_info)
+{
+ struct cxl_pmu_info *info = _info;
+
+ perf_pmu_unregister(&info->pmu);
+}
+
+static void cxl_pmu_cpuhp_remove(void *_info)
+{
+ struct cxl_pmu_info *info = _info;
+
+ cpuhp_state_remove_instance_nocalls(cxl_pmu_cpuhp_state_num, &info->node);
+}
+
+static int cxl_pmu_probe(struct device *dev)
+{
+ struct cxl_pmu *pmu = to_cxl_pmu(dev);
+ struct pci_dev *pdev = to_pci_dev(dev->parent);
+ struct cxl_pmu_info *info;
+ char *irq_name;
+ char *dev_name;
+ int rc, irq;
+
+ info = devm_kzalloc(dev, sizeof(*info), GFP_KERNEL);
+ if (!info)
+ return -ENOMEM;
+
+ dev_set_drvdata(dev, info);
+ INIT_LIST_HEAD(&info->event_caps_fixed);
+ INIT_LIST_HEAD(&info->event_caps_configurable);
+
+ info->base = pmu->base;
+
+ info->on_cpu = -1;
+ rc = cxl_pmu_parse_caps(dev, info);
+ if (rc)
+ return rc;
+
+ info->hw_events = devm_kcalloc(dev, sizeof(*info->hw_events),
+ info->num_counters, GFP_KERNEL);
+ if (!info->hw_events)
+ return -ENOMEM;
+
+ switch (pmu->type) {
+ case CXL_PMU_MEMDEV:
+ dev_name = devm_kasprintf(dev, GFP_KERNEL, "cxl_pmu_mem%d.%d",
+ pmu->assoc_id, pmu->index);
+ break;
+ }
+ if (!dev_name)
+ return -ENOMEM;
+
+ info->pmu = (struct pmu) {
+ .name = dev_name,
+ .parent = dev,
+ .module = THIS_MODULE,
+ .event_init = cxl_pmu_event_init,
+ .pmu_enable = cxl_pmu_enable,
+ .pmu_disable = cxl_pmu_disable,
+ .add = cxl_pmu_event_add,
+ .del = cxl_pmu_event_del,
+ .start = cxl_pmu_event_start,
+ .stop = cxl_pmu_event_stop,
+ .read = cxl_pmu_read,
+ .task_ctx_nr = perf_invalid_context,
+ .attr_groups = cxl_pmu_attr_groups,
+ .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
+ };
+
+ if (info->irq <= 0)
+ return -EINVAL;
+
+ rc = pci_irq_vector(pdev, info->irq);
+ if (rc < 0)
+ return rc;
+ irq = rc;
+
+ irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s_overflow\n", dev_name);
+ if (!irq_name)
+ return -ENOMEM;
+
+ rc = devm_request_irq(dev, irq, cxl_pmu_irq, IRQF_SHARED | IRQF_ONESHOT,
+ irq_name, info);
+ if (rc)
+ return rc;
+ info->irq = irq;
+
+ rc = cpuhp_state_add_instance(cxl_pmu_cpuhp_state_num, &info->node);
+ if (rc)
+ return rc;
+
+ rc = devm_add_action_or_reset(dev, cxl_pmu_cpuhp_remove, info);
+ if (rc)
+ return rc;
+
+ rc = perf_pmu_register(&info->pmu, info->pmu.name, -1);
+ if (rc)
+ return rc;
+
+ rc = devm_add_action_or_reset(dev, cxl_pmu_perf_unregister, info);
+ if (rc)
+ return rc;
+
+ return 0;
+}
+
+static struct cxl_driver cxl_pmu_driver = {
+ .name = "cxl_pmu",
+ .probe = cxl_pmu_probe,
+ .id = CXL_DEVICE_PMU,
+};
+
+static int cxl_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
+{
+ struct cxl_pmu_info *info = hlist_entry_safe(node, struct cxl_pmu_info, node);
+
+ if (info->on_cpu != -1)
+ return 0;
+
+ info->on_cpu = cpu;
+ /*
+ * CPU HP lock is held so we should be guaranteed that the CPU hasn't yet
+ * gone away again.
+ */
+ WARN_ON(irq_set_affinity(info->irq, cpumask_of(cpu)));
+
+ return 0;
+}
+
+static int cxl_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
+{
+ struct cxl_pmu_info *info = hlist_entry_safe(node, struct cxl_pmu_info, node);
+ unsigned int target;
+
+ if (info->on_cpu != cpu)
+ return 0;
+
+ info->on_cpu = -1;
+ target = cpumask_any_but(cpu_online_mask, cpu);
+ if (target >= nr_cpu_ids) {
+ dev_err(info->pmu.dev, "Unable to find a suitable CPU\n");
+ return 0;
+ }
+
+ perf_pmu_migrate_context(&info->pmu, cpu, target);
+ info->on_cpu = target;
+ /*
+ * CPU HP lock is held so we should be guaranteed that this CPU hasn't yet
+ * gone away.
+ */
+ WARN_ON(irq_set_affinity(info->irq, cpumask_of(target)));
+
+ return 0;
+}
+
+static __init int cxl_pmu_init(void)
+{
+ int rc;
+
+ rc = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
+ "AP_PERF_CXL_PMU_ONLINE",
+ cxl_pmu_online_cpu, cxl_pmu_offline_cpu);
+ if (rc < 0)
+ return rc;
+ cxl_pmu_cpuhp_state_num = rc;
+
+ rc = cxl_driver_register(&cxl_pmu_driver);
+ if (rc)
+ cpuhp_remove_multi_state(cxl_pmu_cpuhp_state_num);
+
+ return rc;
+}
+
+static __exit void cxl_pmu_exit(void)
+{
+ cxl_driver_unregister(&cxl_pmu_driver);
+ cpuhp_remove_multi_state(cxl_pmu_cpuhp_state_num);
+}
+
+MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS(CXL);
+module_init(cxl_pmu_init);
+module_exit(cxl_pmu_exit);
+MODULE_ALIAS_CXL(CXL_DEVICE_PMU);
diff --git a/drivers/s390/block/dcssblk.c b/drivers/s390/block/dcssblk.c
index 200f88f..405d76d 100644
--- a/drivers/s390/block/dcssblk.c
+++ b/drivers/s390/block/dcssblk.c
@@ -54,7 +54,8 @@ static int dcssblk_dax_zero_page_range(struct dax_device *dax_dev,
rc = dax_direct_access(dax_dev, pgoff, nr_pages, DAX_ACCESS,
&kaddr, NULL);
if (rc < 0)
- return rc;
+ return dax_mem2blk_err(rc);
+
memset(kaddr, 0, nr_pages << PAGE_SHIFT);
dax_flush(dax_dev, kaddr, nr_pages << PAGE_SHIFT);
return 0;
diff --git a/fs/dax.c b/fs/dax.c
index 2ababb8..906ecbd 100644
--- a/fs/dax.c
+++ b/fs/dax.c
@@ -1148,7 +1148,7 @@ static int dax_iomap_copy_around(loff_t pos, uint64_t length, size_t align_size,
if (!zero_edge) {
ret = dax_iomap_direct_access(srcmap, pos, size, &saddr, NULL);
if (ret)
- return ret;
+ return dax_mem2blk_err(ret);
}
if (copy_all) {
@@ -1310,7 +1310,7 @@ static s64 dax_unshare_iter(struct iomap_iter *iter)
out_unlock:
dax_read_unlock(id);
- return ret;
+ return dax_mem2blk_err(ret);
}
int dax_file_unshare(struct inode *inode, loff_t pos, loff_t len,
@@ -1342,7 +1342,8 @@ static int dax_memzero(struct iomap_iter *iter, loff_t pos, size_t size)
ret = dax_direct_access(iomap->dax_dev, pgoff, 1, DAX_ACCESS, &kaddr,
NULL);
if (ret < 0)
- return ret;
+ return dax_mem2blk_err(ret);
+
memset(kaddr + offset, 0, size);
if (iomap->flags & IOMAP_F_SHARED)
ret = dax_iomap_copy_around(pos, size, PAGE_SIZE, srcmap,
@@ -1498,7 +1499,7 @@ static loff_t dax_iomap_iter(const struct iomap_iter *iomi,
map_len = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size),
DAX_ACCESS, &kaddr, NULL);
- if (map_len == -EIO && iov_iter_rw(iter) == WRITE) {
+ if (map_len == -EHWPOISON && iov_iter_rw(iter) == WRITE) {
map_len = dax_direct_access(dax_dev, pgoff,
PHYS_PFN(size), DAX_RECOVERY_WRITE,
&kaddr, NULL);
@@ -1506,7 +1507,7 @@ static loff_t dax_iomap_iter(const struct iomap_iter *iomi,
recovery = true;
}
if (map_len < 0) {
- ret = map_len;
+ ret = dax_mem2blk_err(map_len);
break;
}
@@ -1830,7 +1831,6 @@ static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp,
vm_fault_t ret = VM_FAULT_FALLBACK;
pgoff_t max_pgoff;
void *entry;
- int error;
if (vmf->flags & FAULT_FLAG_WRITE)
iter.flags |= IOMAP_WRITE;
@@ -1877,7 +1877,7 @@ static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp,
}
iter.pos = (loff_t)xas.xa_index << PAGE_SHIFT;
- while ((error = iomap_iter(&iter, ops)) > 0) {
+ while (iomap_iter(&iter, ops) > 0) {
if (iomap_length(&iter) < PMD_SIZE)
continue; /* actually breaks out of the loop */
diff --git a/fs/fuse/virtio_fs.c b/fs/fuse/virtio_fs.c
index 23a42d8..978c84d 100644
--- a/fs/fuse/virtio_fs.c
+++ b/fs/fuse/virtio_fs.c
@@ -775,7 +775,8 @@ static int virtio_fs_zero_page_range(struct dax_device *dax_dev,
rc = dax_direct_access(dax_dev, pgoff, nr_pages, DAX_ACCESS, &kaddr,
NULL);
if (rc < 0)
- return rc;
+ return dax_mem2blk_err(rc);
+
memset(kaddr, 0, nr_pages << PAGE_SHIFT);
dax_flush(dax_dev, kaddr, nr_pages << PAGE_SHIFT);
return 0;
diff --git a/include/linux/dax.h b/include/linux/dax.h
index bf62584..261944e 100644
--- a/include/linux/dax.h
+++ b/include/linux/dax.h
@@ -261,6 +261,19 @@ static inline bool dax_mapping(struct address_space *mapping)
return mapping->host && IS_DAX(mapping->host);
}
+/*
+ * Due to dax's memory and block duo personalities, hwpoison reporting
+ * takes into consideration which personality is presently visible.
+ * When dax acts like a block device, such as in block IO, an encounter of
+ * dax hwpoison is reported as -EIO.
+ * When dax acts like memory, such as in page fault, a detection of hwpoison
+ * is reported as -EHWPOISON which leads to VM_FAULT_HWPOISON.
+ */
+static inline int dax_mem2blk_err(int err)
+{
+ return (err == -EHWPOISON) ? -EIO : err;
+}
+
#ifdef CONFIG_DEV_DAX_HMEM_DEVICES
void hmem_register_resource(int target_nid, struct resource *r);
#else
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 703ba82..74f1be74 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -3379,6 +3379,8 @@ static inline vm_fault_t vmf_error(int err)
{
if (err == -ENOMEM)
return VM_FAULT_OOM;
+ else if (err == -EHWPOISON)
+ return VM_FAULT_HWPOISON;
return VM_FAULT_SIGBUS;
}
diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index 34f37be..e32ae8b 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -305,6 +305,7 @@ struct pmu {
struct module *module;
struct device *dev;
+ struct device *parent;
const struct attribute_group **attr_groups;
const struct attribute_group **attr_update;
const char *name;
diff --git a/include/linux/rcuwait.h b/include/linux/rcuwait.h
index 8052d34d..2734342 100644
--- a/include/linux/rcuwait.h
+++ b/include/linux/rcuwait.h
@@ -49,9 +49,9 @@ static inline void prepare_to_rcuwait(struct rcuwait *w)
extern void finish_rcuwait(struct rcuwait *w);
-#define rcuwait_wait_event(w, condition, state) \
+#define ___rcuwait_wait_event(w, condition, state, ret, cmd) \
({ \
- int __ret = 0; \
+ long __ret = ret; \
prepare_to_rcuwait(w); \
for (;;) { \
/* \
@@ -67,10 +67,27 @@ extern void finish_rcuwait(struct rcuwait *w);
break; \
} \
\
- schedule(); \
+ cmd; \
} \
finish_rcuwait(w); \
__ret; \
})
+#define rcuwait_wait_event(w, condition, state) \
+ ___rcuwait_wait_event(w, condition, state, 0, schedule())
+
+#define __rcuwait_wait_event_timeout(w, condition, state, timeout) \
+ ___rcuwait_wait_event(w, ___wait_cond_timeout(condition), \
+ state, timeout, \
+ __ret = schedule_timeout(__ret))
+
+#define rcuwait_wait_event_timeout(w, condition, state, timeout) \
+({ \
+ long __ret = timeout; \
+ if (!___wait_cond_timeout(condition)) \
+ __ret = __rcuwait_wait_event_timeout(w, condition, \
+ state, timeout); \
+ __ret; \
+})
+
#endif /* _LINUX_RCUWAIT_H_ */
diff --git a/kernel/events/core.c b/kernel/events/core.c
index 76cc068..60eedea 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -11377,6 +11377,7 @@ static int pmu_dev_alloc(struct pmu *pmu)
dev_set_drvdata(pmu->dev, pmu);
pmu->dev->bus = &pmu_bus;
+ pmu->dev->parent = pmu->parent;
pmu->dev->release = pmu_dev_release;
ret = dev_set_name(pmu->dev, "%s", pmu->name);
diff --git a/tools/testing/cxl/Kbuild b/tools/testing/cxl/Kbuild
index 6f9347a..90f3c98 100644
--- a/tools/testing/cxl/Kbuild
+++ b/tools/testing/cxl/Kbuild
@@ -6,13 +6,13 @@
ldflags-y += --wrap=nvdimm_bus_register
ldflags-y += --wrap=devm_cxl_port_enumerate_dports
ldflags-y += --wrap=devm_cxl_setup_hdm
-ldflags-y += --wrap=devm_cxl_enable_hdm
ldflags-y += --wrap=devm_cxl_add_passthrough_decoder
ldflags-y += --wrap=devm_cxl_enumerate_decoders
ldflags-y += --wrap=cxl_await_media_ready
ldflags-y += --wrap=cxl_hdm_decode_init
ldflags-y += --wrap=cxl_dvsec_rr_decode
-ldflags-y += --wrap=cxl_rcrb_to_component
+ldflags-y += --wrap=devm_cxl_add_rch_dport
+ldflags-y += --wrap=cxl_rcd_component_reg_phys
DRIVERS := ../../../drivers
CXL_SRC := $(DRIVERS)/cxl
@@ -57,6 +57,7 @@
cxl_core-y += $(CXL_CORE_SRC)/mbox.o
cxl_core-y += $(CXL_CORE_SRC)/pci.o
cxl_core-y += $(CXL_CORE_SRC)/hdm.o
+cxl_core-y += $(CXL_CORE_SRC)/pmu.o
cxl_core-$(CONFIG_TRACING) += $(CXL_CORE_SRC)/trace.o
cxl_core-$(CONFIG_CXL_REGION) += $(CXL_CORE_SRC)/region.o
cxl_core-y += config_check.o
diff --git a/tools/testing/cxl/test/cxl.c b/tools/testing/cxl/test/cxl.c
index bf00dc5..0e78d8e 100644
--- a/tools/testing/cxl/test/cxl.c
+++ b/tools/testing/cxl/test/cxl.c
@@ -713,7 +713,7 @@ static void default_mock_decoder(struct cxl_decoder *cxld)
cxld->interleave_ways = 1;
cxld->interleave_granularity = 256;
- cxld->target_type = CXL_DECODER_EXPANDER;
+ cxld->target_type = CXL_DECODER_HOSTONLYMEM;
cxld->commit = mock_decoder_commit;
cxld->reset = mock_decoder_reset;
}
@@ -754,7 +754,7 @@ static void mock_init_hdm_decoder(struct cxl_decoder *cxld)
/* check is endpoint is attach to host-bridge0 */
port = cxled_to_port(cxled);
do {
- if (port->uport == &cxl_host_bridge[0]->dev) {
+ if (port->uport_dev == &cxl_host_bridge[0]->dev) {
hb0 = true;
break;
}
@@ -787,7 +787,7 @@ static void mock_init_hdm_decoder(struct cxl_decoder *cxld)
cxld->interleave_ways = 2;
eig_to_granularity(window->granularity, &cxld->interleave_granularity);
- cxld->target_type = CXL_DECODER_EXPANDER;
+ cxld->target_type = CXL_DECODER_HOSTONLYMEM;
cxld->flags = CXL_DECODER_F_ENABLE;
cxled->state = CXL_DECODER_STATE_AUTO;
port->commit_end = cxld->id;
@@ -820,7 +820,7 @@ static void mock_init_hdm_decoder(struct cxl_decoder *cxld)
} else
cxlsd->target[0] = dport;
cxld = &cxlsd->cxld;
- cxld->target_type = CXL_DECODER_EXPANDER;
+ cxld->target_type = CXL_DECODER_HOSTONLYMEM;
cxld->flags = CXL_DECODER_F_ENABLE;
iter->commit_end = 0;
/*
@@ -889,7 +889,7 @@ static int mock_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm,
mock_init_hdm_decoder(cxld);
if (target_count) {
- rc = device_for_each_child(port->uport, &ctx,
+ rc = device_for_each_child(port->uport_dev, &ctx,
map_targets);
if (rc) {
put_device(&cxld->dev);
@@ -919,29 +919,29 @@ static int mock_cxl_port_enumerate_dports(struct cxl_port *port)
int i, array_size;
if (port->depth == 1) {
- if (is_multi_bridge(port->uport)) {
+ if (is_multi_bridge(port->uport_dev)) {
array_size = ARRAY_SIZE(cxl_root_port);
array = cxl_root_port;
- } else if (is_single_bridge(port->uport)) {
+ } else if (is_single_bridge(port->uport_dev)) {
array_size = ARRAY_SIZE(cxl_root_single);
array = cxl_root_single;
} else {
dev_dbg(&port->dev, "%s: unknown bridge type\n",
- dev_name(port->uport));
+ dev_name(port->uport_dev));
return -ENXIO;
}
} else if (port->depth == 2) {
struct cxl_port *parent = to_cxl_port(port->dev.parent);
- if (is_multi_bridge(parent->uport)) {
+ if (is_multi_bridge(parent->uport_dev)) {
array_size = ARRAY_SIZE(cxl_switch_dport);
array = cxl_switch_dport;
- } else if (is_single_bridge(parent->uport)) {
+ } else if (is_single_bridge(parent->uport_dev)) {
array_size = ARRAY_SIZE(cxl_swd_single);
array = cxl_swd_single;
} else {
dev_dbg(&port->dev, "%s: unknown bridge type\n",
- dev_name(port->uport));
+ dev_name(port->uport_dev));
return -ENXIO;
}
} else {
@@ -954,9 +954,9 @@ static int mock_cxl_port_enumerate_dports(struct cxl_port *port)
struct platform_device *pdev = array[i];
struct cxl_dport *dport;
- if (pdev->dev.parent != port->uport) {
+ if (pdev->dev.parent != port->uport_dev) {
dev_dbg(&port->dev, "%s: mismatch parent %s\n",
- dev_name(port->uport),
+ dev_name(port->uport_dev),
dev_name(pdev->dev.parent));
continue;
}
@@ -971,15 +971,6 @@ static int mock_cxl_port_enumerate_dports(struct cxl_port *port)
return 0;
}
-resource_size_t mock_cxl_rcrb_to_component(struct device *dev,
- resource_size_t rcrb,
- enum cxl_rcrb which)
-{
- dev_dbg(dev, "rcrb: %pa which: %d\n", &rcrb, which);
-
- return (resource_size_t) which + 1;
-}
-
static struct cxl_mock_ops cxl_mock_ops = {
.is_mock_adev = is_mock_adev,
.is_mock_bridge = is_mock_bridge,
@@ -988,7 +979,6 @@ static struct cxl_mock_ops cxl_mock_ops = {
.is_mock_dev = is_mock_dev,
.acpi_table_parse_cedt = mock_acpi_table_parse_cedt,
.acpi_evaluate_integer = mock_acpi_evaluate_integer,
- .cxl_rcrb_to_component = mock_cxl_rcrb_to_component,
.acpi_pci_find_root = mock_acpi_pci_find_root,
.devm_cxl_port_enumerate_dports = mock_cxl_port_enumerate_dports,
.devm_cxl_setup_hdm = mock_cxl_setup_hdm,
diff --git a/tools/testing/cxl/test/mem.c b/tools/testing/cxl/test/mem.c
index 34b4802..464fc39 100644
--- a/tools/testing/cxl/test/mem.c
+++ b/tools/testing/cxl/test/mem.c
@@ -8,11 +8,14 @@
#include <linux/sizes.h>
#include <linux/bits.h>
#include <asm/unaligned.h>
+#include <crypto/sha2.h>
#include <cxlmem.h>
#include "trace.h"
#define LSA_SIZE SZ_128K
+#define FW_SIZE SZ_64M
+#define FW_SLOTS 3
#define DEV_SIZE SZ_2G
#define EFFECT(x) (1U << x)
@@ -21,42 +24,70 @@
static unsigned int poison_inject_dev_max = MOCK_INJECT_DEV_MAX;
+enum cxl_command_effects {
+ CONF_CHANGE_COLD_RESET = 0,
+ CONF_CHANGE_IMMEDIATE,
+ DATA_CHANGE_IMMEDIATE,
+ POLICY_CHANGE_IMMEDIATE,
+ LOG_CHANGE_IMMEDIATE,
+ SECURITY_CHANGE_IMMEDIATE,
+ BACKGROUND_OP,
+ SECONDARY_MBOX_SUPPORTED,
+};
+
+#define CXL_CMD_EFFECT_NONE cpu_to_le16(0)
+
static struct cxl_cel_entry mock_cel[] = {
{
.opcode = cpu_to_le16(CXL_MBOX_OP_GET_SUPPORTED_LOGS),
- .effect = cpu_to_le16(0),
+ .effect = CXL_CMD_EFFECT_NONE,
},
{
.opcode = cpu_to_le16(CXL_MBOX_OP_IDENTIFY),
- .effect = cpu_to_le16(0),
+ .effect = CXL_CMD_EFFECT_NONE,
},
{
.opcode = cpu_to_le16(CXL_MBOX_OP_GET_LSA),
- .effect = cpu_to_le16(0),
+ .effect = CXL_CMD_EFFECT_NONE,
},
{
.opcode = cpu_to_le16(CXL_MBOX_OP_GET_PARTITION_INFO),
- .effect = cpu_to_le16(0),
+ .effect = CXL_CMD_EFFECT_NONE,
},
{
.opcode = cpu_to_le16(CXL_MBOX_OP_SET_LSA),
- .effect = cpu_to_le16(EFFECT(1) | EFFECT(2)),
+ .effect = cpu_to_le16(EFFECT(CONF_CHANGE_IMMEDIATE) |
+ EFFECT(DATA_CHANGE_IMMEDIATE)),
},
{
.opcode = cpu_to_le16(CXL_MBOX_OP_GET_HEALTH_INFO),
- .effect = cpu_to_le16(0),
+ .effect = CXL_CMD_EFFECT_NONE,
},
{
.opcode = cpu_to_le16(CXL_MBOX_OP_GET_POISON),
- .effect = cpu_to_le16(0),
+ .effect = CXL_CMD_EFFECT_NONE,
},
{
.opcode = cpu_to_le16(CXL_MBOX_OP_INJECT_POISON),
- .effect = cpu_to_le16(0),
+ .effect = cpu_to_le16(EFFECT(DATA_CHANGE_IMMEDIATE)),
},
{
.opcode = cpu_to_le16(CXL_MBOX_OP_CLEAR_POISON),
- .effect = cpu_to_le16(0),
+ .effect = cpu_to_le16(EFFECT(DATA_CHANGE_IMMEDIATE)),
+ },
+ {
+ .opcode = cpu_to_le16(CXL_MBOX_OP_GET_FW_INFO),
+ .effect = CXL_CMD_EFFECT_NONE,
+ },
+ {
+ .opcode = cpu_to_le16(CXL_MBOX_OP_TRANSFER_FW),
+ .effect = cpu_to_le16(EFFECT(CONF_CHANGE_COLD_RESET) |
+ EFFECT(BACKGROUND_OP)),
+ },
+ {
+ .opcode = cpu_to_le16(CXL_MBOX_OP_ACTIVATE_FW),
+ .effect = cpu_to_le16(EFFECT(CONF_CHANGE_COLD_RESET) |
+ EFFECT(CONF_CHANGE_IMMEDIATE)),
},
};
@@ -102,13 +133,17 @@ struct mock_event_log {
};
struct mock_event_store {
- struct cxl_dev_state *cxlds;
+ struct cxl_memdev_state *mds;
struct mock_event_log mock_logs[CXL_EVENT_TYPE_MAX];
u32 ev_status;
};
struct cxl_mockmem_data {
void *lsa;
+ void *fw;
+ int fw_slot;
+ int fw_staged;
+ size_t fw_size;
u32 security_state;
u8 user_pass[NVDIMM_PASSPHRASE_LEN];
u8 master_pass[NVDIMM_PASSPHRASE_LEN];
@@ -180,8 +215,7 @@ static void mes_add_event(struct mock_event_store *mes,
log->nr_events++;
}
-static int mock_get_event(struct cxl_dev_state *cxlds,
- struct cxl_mbox_cmd *cmd)
+static int mock_get_event(struct device *dev, struct cxl_mbox_cmd *cmd)
{
struct cxl_get_event_payload *pl;
struct mock_event_log *log;
@@ -201,7 +235,7 @@ static int mock_get_event(struct cxl_dev_state *cxlds,
memset(cmd->payload_out, 0, cmd->size_out);
- log = event_find_log(cxlds->dev, log_type);
+ log = event_find_log(dev, log_type);
if (!log || event_log_empty(log))
return 0;
@@ -234,8 +268,7 @@ static int mock_get_event(struct cxl_dev_state *cxlds,
return 0;
}
-static int mock_clear_event(struct cxl_dev_state *cxlds,
- struct cxl_mbox_cmd *cmd)
+static int mock_clear_event(struct device *dev, struct cxl_mbox_cmd *cmd)
{
struct cxl_mbox_clear_event_payload *pl = cmd->payload_in;
struct mock_event_log *log;
@@ -246,7 +279,7 @@ static int mock_clear_event(struct cxl_dev_state *cxlds,
if (log_type >= CXL_EVENT_TYPE_MAX)
return -EINVAL;
- log = event_find_log(cxlds->dev, log_type);
+ log = event_find_log(dev, log_type);
if (!log)
return 0; /* No mock data in this log */
@@ -256,7 +289,7 @@ static int mock_clear_event(struct cxl_dev_state *cxlds,
* However, this is not good behavior for the host so test it.
*/
if (log->clear_idx + pl->nr_recs > log->cur_idx) {
- dev_err(cxlds->dev,
+ dev_err(dev,
"Attempting to clear more events than returned!\n");
return -EINVAL;
}
@@ -266,7 +299,7 @@ static int mock_clear_event(struct cxl_dev_state *cxlds,
nr < pl->nr_recs;
nr++, handle++) {
if (handle != le16_to_cpu(pl->handles[nr])) {
- dev_err(cxlds->dev, "Clearing events out of order\n");
+ dev_err(dev, "Clearing events out of order\n");
return -EINVAL;
}
}
@@ -293,7 +326,7 @@ static void cxl_mock_event_trigger(struct device *dev)
event_reset_log(log);
}
- cxl_mem_get_event_records(mes->cxlds, mes->ev_status);
+ cxl_mem_get_event_records(mes->mds, mes->ev_status);
}
struct cxl_event_record_raw maint_needed = {
@@ -453,7 +486,7 @@ static int mock_gsl(struct cxl_mbox_cmd *cmd)
return 0;
}
-static int mock_get_log(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd *cmd)
+static int mock_get_log(struct cxl_memdev_state *mds, struct cxl_mbox_cmd *cmd)
{
struct cxl_mbox_get_log *gl = cmd->payload_in;
u32 offset = le32_to_cpu(gl->offset);
@@ -463,7 +496,7 @@ static int mock_get_log(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd *cmd)
if (cmd->size_in < sizeof(*gl))
return -EINVAL;
- if (length > cxlds->payload_size)
+ if (length > mds->payload_size)
return -EINVAL;
if (offset + length > sizeof(mock_cel))
return -EINVAL;
@@ -477,7 +510,7 @@ static int mock_get_log(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd *cmd)
return 0;
}
-static int mock_rcd_id(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd *cmd)
+static int mock_rcd_id(struct cxl_mbox_cmd *cmd)
{
struct cxl_mbox_identify id = {
.fw_revision = { "mock fw v1 " },
@@ -495,7 +528,7 @@ static int mock_rcd_id(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd *cmd)
return 0;
}
-static int mock_id(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd *cmd)
+static int mock_id(struct cxl_mbox_cmd *cmd)
{
struct cxl_mbox_identify id = {
.fw_revision = { "mock fw v1 " },
@@ -517,8 +550,7 @@ static int mock_id(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd *cmd)
return 0;
}
-static int mock_partition_info(struct cxl_dev_state *cxlds,
- struct cxl_mbox_cmd *cmd)
+static int mock_partition_info(struct cxl_mbox_cmd *cmd)
{
struct cxl_mbox_get_partition_info pi = {
.active_volatile_cap =
@@ -535,11 +567,52 @@ static int mock_partition_info(struct cxl_dev_state *cxlds,
return 0;
}
-static int mock_get_security_state(struct cxl_dev_state *cxlds,
+static int mock_sanitize(struct cxl_mockmem_data *mdata,
+ struct cxl_mbox_cmd *cmd)
+{
+ if (cmd->size_in != 0)
+ return -EINVAL;
+
+ if (cmd->size_out != 0)
+ return -EINVAL;
+
+ if (mdata->security_state & CXL_PMEM_SEC_STATE_USER_PASS_SET) {
+ cmd->return_code = CXL_MBOX_CMD_RC_SECURITY;
+ return -ENXIO;
+ }
+ if (mdata->security_state & CXL_PMEM_SEC_STATE_LOCKED) {
+ cmd->return_code = CXL_MBOX_CMD_RC_SECURITY;
+ return -ENXIO;
+ }
+
+ return 0; /* assume less than 2 secs, no bg */
+}
+
+static int mock_secure_erase(struct cxl_mockmem_data *mdata,
+ struct cxl_mbox_cmd *cmd)
+{
+ if (cmd->size_in != 0)
+ return -EINVAL;
+
+ if (cmd->size_out != 0)
+ return -EINVAL;
+
+ if (mdata->security_state & CXL_PMEM_SEC_STATE_USER_PASS_SET) {
+ cmd->return_code = CXL_MBOX_CMD_RC_SECURITY;
+ return -ENXIO;
+ }
+
+ if (mdata->security_state & CXL_PMEM_SEC_STATE_LOCKED) {
+ cmd->return_code = CXL_MBOX_CMD_RC_SECURITY;
+ return -ENXIO;
+ }
+
+ return 0;
+}
+
+static int mock_get_security_state(struct cxl_mockmem_data *mdata,
struct cxl_mbox_cmd *cmd)
{
- struct cxl_mockmem_data *mdata = dev_get_drvdata(cxlds->dev);
-
if (cmd->size_in)
return -EINVAL;
@@ -569,9 +642,9 @@ static void user_plimit_check(struct cxl_mockmem_data *mdata)
mdata->security_state |= CXL_PMEM_SEC_STATE_USER_PLIMIT;
}
-static int mock_set_passphrase(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd *cmd)
+static int mock_set_passphrase(struct cxl_mockmem_data *mdata,
+ struct cxl_mbox_cmd *cmd)
{
- struct cxl_mockmem_data *mdata = dev_get_drvdata(cxlds->dev);
struct cxl_set_pass *set_pass;
if (cmd->size_in != sizeof(*set_pass))
@@ -629,9 +702,9 @@ static int mock_set_passphrase(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd
return -EINVAL;
}
-static int mock_disable_passphrase(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd *cmd)
+static int mock_disable_passphrase(struct cxl_mockmem_data *mdata,
+ struct cxl_mbox_cmd *cmd)
{
- struct cxl_mockmem_data *mdata = dev_get_drvdata(cxlds->dev);
struct cxl_disable_pass *dis_pass;
if (cmd->size_in != sizeof(*dis_pass))
@@ -700,10 +773,9 @@ static int mock_disable_passphrase(struct cxl_dev_state *cxlds, struct cxl_mbox_
return 0;
}
-static int mock_freeze_security(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd *cmd)
+static int mock_freeze_security(struct cxl_mockmem_data *mdata,
+ struct cxl_mbox_cmd *cmd)
{
- struct cxl_mockmem_data *mdata = dev_get_drvdata(cxlds->dev);
-
if (cmd->size_in != 0)
return -EINVAL;
@@ -717,10 +789,9 @@ static int mock_freeze_security(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd
return 0;
}
-static int mock_unlock_security(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd *cmd)
+static int mock_unlock_security(struct cxl_mockmem_data *mdata,
+ struct cxl_mbox_cmd *cmd)
{
- struct cxl_mockmem_data *mdata = dev_get_drvdata(cxlds->dev);
-
if (cmd->size_in != NVDIMM_PASSPHRASE_LEN)
return -EINVAL;
@@ -759,10 +830,9 @@ static int mock_unlock_security(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd
return 0;
}
-static int mock_passphrase_secure_erase(struct cxl_dev_state *cxlds,
+static int mock_passphrase_secure_erase(struct cxl_mockmem_data *mdata,
struct cxl_mbox_cmd *cmd)
{
- struct cxl_mockmem_data *mdata = dev_get_drvdata(cxlds->dev);
struct cxl_pass_erase *erase;
if (cmd->size_in != sizeof(*erase))
@@ -858,10 +928,10 @@ static int mock_passphrase_secure_erase(struct cxl_dev_state *cxlds,
return 0;
}
-static int mock_get_lsa(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd *cmd)
+static int mock_get_lsa(struct cxl_mockmem_data *mdata,
+ struct cxl_mbox_cmd *cmd)
{
struct cxl_mbox_get_lsa *get_lsa = cmd->payload_in;
- struct cxl_mockmem_data *mdata = dev_get_drvdata(cxlds->dev);
void *lsa = mdata->lsa;
u32 offset, length;
@@ -878,10 +948,10 @@ static int mock_get_lsa(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd *cmd)
return 0;
}
-static int mock_set_lsa(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd *cmd)
+static int mock_set_lsa(struct cxl_mockmem_data *mdata,
+ struct cxl_mbox_cmd *cmd)
{
struct cxl_mbox_set_lsa *set_lsa = cmd->payload_in;
- struct cxl_mockmem_data *mdata = dev_get_drvdata(cxlds->dev);
void *lsa = mdata->lsa;
u32 offset, length;
@@ -896,8 +966,7 @@ static int mock_set_lsa(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd *cmd)
return 0;
}
-static int mock_health_info(struct cxl_dev_state *cxlds,
- struct cxl_mbox_cmd *cmd)
+static int mock_health_info(struct cxl_mbox_cmd *cmd)
{
struct cxl_mbox_health_info health_info = {
/* set flags for maint needed, perf degraded, hw replacement */
@@ -1114,9 +1183,90 @@ static struct attribute *cxl_mock_mem_core_attrs[] = {
};
ATTRIBUTE_GROUPS(cxl_mock_mem_core);
-static int cxl_mock_mbox_send(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd *cmd)
+static int mock_fw_info(struct cxl_mockmem_data *mdata,
+ struct cxl_mbox_cmd *cmd)
{
+ struct cxl_mbox_get_fw_info fw_info = {
+ .num_slots = FW_SLOTS,
+ .slot_info = (mdata->fw_slot & 0x7) |
+ ((mdata->fw_staged & 0x7) << 3),
+ .activation_cap = 0,
+ };
+
+ strcpy(fw_info.slot_1_revision, "cxl_test_fw_001");
+ strcpy(fw_info.slot_2_revision, "cxl_test_fw_002");
+ strcpy(fw_info.slot_3_revision, "cxl_test_fw_003");
+ strcpy(fw_info.slot_4_revision, "");
+
+ if (cmd->size_out < sizeof(fw_info))
+ return -EINVAL;
+
+ memcpy(cmd->payload_out, &fw_info, sizeof(fw_info));
+ return 0;
+}
+
+static int mock_transfer_fw(struct cxl_mockmem_data *mdata,
+ struct cxl_mbox_cmd *cmd)
+{
+ struct cxl_mbox_transfer_fw *transfer = cmd->payload_in;
+ void *fw = mdata->fw;
+ size_t offset, length;
+
+ offset = le32_to_cpu(transfer->offset) * CXL_FW_TRANSFER_ALIGNMENT;
+ length = cmd->size_in - sizeof(*transfer);
+ if (offset + length > FW_SIZE)
+ return -EINVAL;
+
+ switch (transfer->action) {
+ case CXL_FW_TRANSFER_ACTION_FULL:
+ if (offset != 0)
+ return -EINVAL;
+ fallthrough;
+ case CXL_FW_TRANSFER_ACTION_END:
+ if (transfer->slot == 0 || transfer->slot > FW_SLOTS)
+ return -EINVAL;
+ mdata->fw_size = offset + length;
+ break;
+ case CXL_FW_TRANSFER_ACTION_INITIATE:
+ case CXL_FW_TRANSFER_ACTION_CONTINUE:
+ break;
+ case CXL_FW_TRANSFER_ACTION_ABORT:
+ return 0;
+ default:
+ return -EINVAL;
+ }
+
+ memcpy(fw + offset, transfer->data, length);
+ return 0;
+}
+
+static int mock_activate_fw(struct cxl_mockmem_data *mdata,
+ struct cxl_mbox_cmd *cmd)
+{
+ struct cxl_mbox_activate_fw *activate = cmd->payload_in;
+
+ if (activate->slot == 0 || activate->slot > FW_SLOTS)
+ return -EINVAL;
+
+ switch (activate->action) {
+ case CXL_FW_ACTIVATE_ONLINE:
+ mdata->fw_slot = activate->slot;
+ mdata->fw_staged = 0;
+ return 0;
+ case CXL_FW_ACTIVATE_OFFLINE:
+ mdata->fw_staged = activate->slot;
+ return 0;
+ }
+
+ return -EINVAL;
+}
+
+static int cxl_mock_mbox_send(struct cxl_memdev_state *mds,
+ struct cxl_mbox_cmd *cmd)
+{
+ struct cxl_dev_state *cxlds = &mds->cxlds;
struct device *dev = cxlds->dev;
+ struct cxl_mockmem_data *mdata = dev_get_drvdata(dev);
int rc = -EIO;
switch (cmd->opcode) {
@@ -1127,49 +1277,55 @@ static int cxl_mock_mbox_send(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd *
rc = mock_gsl(cmd);
break;
case CXL_MBOX_OP_GET_LOG:
- rc = mock_get_log(cxlds, cmd);
+ rc = mock_get_log(mds, cmd);
break;
case CXL_MBOX_OP_IDENTIFY:
if (cxlds->rcd)
- rc = mock_rcd_id(cxlds, cmd);
+ rc = mock_rcd_id(cmd);
else
- rc = mock_id(cxlds, cmd);
+ rc = mock_id(cmd);
break;
case CXL_MBOX_OP_GET_LSA:
- rc = mock_get_lsa(cxlds, cmd);
+ rc = mock_get_lsa(mdata, cmd);
break;
case CXL_MBOX_OP_GET_PARTITION_INFO:
- rc = mock_partition_info(cxlds, cmd);
+ rc = mock_partition_info(cmd);
break;
case CXL_MBOX_OP_GET_EVENT_RECORD:
- rc = mock_get_event(cxlds, cmd);
+ rc = mock_get_event(dev, cmd);
break;
case CXL_MBOX_OP_CLEAR_EVENT_RECORD:
- rc = mock_clear_event(cxlds, cmd);
+ rc = mock_clear_event(dev, cmd);
break;
case CXL_MBOX_OP_SET_LSA:
- rc = mock_set_lsa(cxlds, cmd);
+ rc = mock_set_lsa(mdata, cmd);
break;
case CXL_MBOX_OP_GET_HEALTH_INFO:
- rc = mock_health_info(cxlds, cmd);
+ rc = mock_health_info(cmd);
+ break;
+ case CXL_MBOX_OP_SANITIZE:
+ rc = mock_sanitize(mdata, cmd);
+ break;
+ case CXL_MBOX_OP_SECURE_ERASE:
+ rc = mock_secure_erase(mdata, cmd);
break;
case CXL_MBOX_OP_GET_SECURITY_STATE:
- rc = mock_get_security_state(cxlds, cmd);
+ rc = mock_get_security_state(mdata, cmd);
break;
case CXL_MBOX_OP_SET_PASSPHRASE:
- rc = mock_set_passphrase(cxlds, cmd);
+ rc = mock_set_passphrase(mdata, cmd);
break;
case CXL_MBOX_OP_DISABLE_PASSPHRASE:
- rc = mock_disable_passphrase(cxlds, cmd);
+ rc = mock_disable_passphrase(mdata, cmd);
break;
case CXL_MBOX_OP_FREEZE_SECURITY:
- rc = mock_freeze_security(cxlds, cmd);
+ rc = mock_freeze_security(mdata, cmd);
break;
case CXL_MBOX_OP_UNLOCK:
- rc = mock_unlock_security(cxlds, cmd);
+ rc = mock_unlock_security(mdata, cmd);
break;
case CXL_MBOX_OP_PASSPHRASE_SECURE_ERASE:
- rc = mock_passphrase_secure_erase(cxlds, cmd);
+ rc = mock_passphrase_secure_erase(mdata, cmd);
break;
case CXL_MBOX_OP_GET_POISON:
rc = mock_get_poison(cxlds, cmd);
@@ -1180,6 +1336,15 @@ static int cxl_mock_mbox_send(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd *
case CXL_MBOX_OP_CLEAR_POISON:
rc = mock_clear_poison(cxlds, cmd);
break;
+ case CXL_MBOX_OP_GET_FW_INFO:
+ rc = mock_fw_info(mdata, cmd);
+ break;
+ case CXL_MBOX_OP_TRANSFER_FW:
+ rc = mock_transfer_fw(mdata, cmd);
+ break;
+ case CXL_MBOX_OP_ACTIVATE_FW:
+ rc = mock_activate_fw(mdata, cmd);
+ break;
default:
break;
}
@@ -1195,6 +1360,11 @@ static void label_area_release(void *lsa)
vfree(lsa);
}
+static void fw_buf_release(void *buf)
+{
+ vfree(buf);
+}
+
static bool is_rcd(struct platform_device *pdev)
{
const struct platform_device_id *id = platform_get_device_id(pdev);
@@ -1215,6 +1385,7 @@ static int cxl_mock_mem_probe(struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
struct cxl_memdev *cxlmd;
+ struct cxl_memdev_state *mds;
struct cxl_dev_state *cxlds;
struct cxl_mockmem_data *mdata;
int rc;
@@ -1227,52 +1398,67 @@ static int cxl_mock_mem_probe(struct platform_device *pdev)
mdata->lsa = vmalloc(LSA_SIZE);
if (!mdata->lsa)
return -ENOMEM;
+ mdata->fw = vmalloc(FW_SIZE);
+ if (!mdata->fw)
+ return -ENOMEM;
+ mdata->fw_slot = 2;
+
rc = devm_add_action_or_reset(dev, label_area_release, mdata->lsa);
if (rc)
return rc;
- cxlds = cxl_dev_state_create(dev);
- if (IS_ERR(cxlds))
- return PTR_ERR(cxlds);
+ rc = devm_add_action_or_reset(dev, fw_buf_release, mdata->fw);
+ if (rc)
+ return rc;
+ mds = cxl_memdev_state_create(dev);
+ if (IS_ERR(mds))
+ return PTR_ERR(mds);
+
+ mds->mbox_send = cxl_mock_mbox_send;
+ mds->payload_size = SZ_4K;
+ mds->event.buf = (struct cxl_get_event_payload *) mdata->event_buf;
+
+ cxlds = &mds->cxlds;
cxlds->serial = pdev->id;
- cxlds->mbox_send = cxl_mock_mbox_send;
- cxlds->payload_size = SZ_4K;
- cxlds->event.buf = (struct cxl_get_event_payload *) mdata->event_buf;
if (is_rcd(pdev)) {
cxlds->rcd = true;
cxlds->component_reg_phys = CXL_RESOURCE_NONE;
}
- rc = cxl_enumerate_cmds(cxlds);
+ rc = cxl_enumerate_cmds(mds);
if (rc)
return rc;
- rc = cxl_poison_state_init(cxlds);
+ rc = cxl_poison_state_init(mds);
if (rc)
return rc;
- rc = cxl_set_timestamp(cxlds);
+ rc = cxl_set_timestamp(mds);
if (rc)
return rc;
cxlds->media_ready = true;
- rc = cxl_dev_state_identify(cxlds);
+ rc = cxl_dev_state_identify(mds);
if (rc)
return rc;
- rc = cxl_mem_create_range_info(cxlds);
+ rc = cxl_mem_create_range_info(mds);
if (rc)
return rc;
- mdata->mes.cxlds = cxlds;
+ mdata->mes.mds = mds;
cxl_mock_add_event_logs(&mdata->mes);
cxlmd = devm_cxl_add_memdev(cxlds);
if (IS_ERR(cxlmd))
return PTR_ERR(cxlmd);
- cxl_mem_get_event_records(cxlds, CXLDEV_EVENT_STATUS_ALL);
+ rc = cxl_memdev_setup_fw_upload(mds);
+ if (rc)
+ return rc;
+
+ cxl_mem_get_event_records(mds, CXLDEV_EVENT_STATUS_ALL);
return 0;
}
@@ -1310,9 +1496,40 @@ static ssize_t security_lock_store(struct device *dev, struct device_attribute *
static DEVICE_ATTR_RW(security_lock);
+static ssize_t fw_buf_checksum_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct cxl_mockmem_data *mdata = dev_get_drvdata(dev);
+ u8 hash[SHA256_DIGEST_SIZE];
+ unsigned char *hstr, *hptr;
+ struct sha256_state sctx;
+ ssize_t written = 0;
+ int i;
+
+ sha256_init(&sctx);
+ sha256_update(&sctx, mdata->fw, mdata->fw_size);
+ sha256_final(&sctx, hash);
+
+ hstr = kzalloc((SHA256_DIGEST_SIZE * 2) + 1, GFP_KERNEL);
+ if (!hstr)
+ return -ENOMEM;
+
+ hptr = hstr;
+ for (i = 0; i < SHA256_DIGEST_SIZE; i++)
+ hptr += sprintf(hptr, "%02x", hash[i]);
+
+ written = sysfs_emit(buf, "%s\n", hstr);
+
+ kfree(hstr);
+ return written;
+}
+
+static DEVICE_ATTR_RO(fw_buf_checksum);
+
static struct attribute *cxl_mock_mem_attrs[] = {
&dev_attr_security_lock.attr,
&dev_attr_event_trigger.attr,
+ &dev_attr_fw_buf_checksum.attr,
NULL
};
ATTRIBUTE_GROUPS(cxl_mock_mem);
diff --git a/tools/testing/cxl/test/mock.c b/tools/testing/cxl/test/mock.c
index 2844165..1a61e68 100644
--- a/tools/testing/cxl/test/mock.c
+++ b/tools/testing/cxl/test/mock.c
@@ -139,7 +139,7 @@ struct cxl_hdm *__wrap_devm_cxl_setup_hdm(struct cxl_port *port,
struct cxl_hdm *cxlhdm;
struct cxl_mock_ops *ops = get_cxl_mock_ops(&index);
- if (ops && ops->is_mock_port(port->uport))
+ if (ops && ops->is_mock_port(port->uport_dev))
cxlhdm = ops->devm_cxl_setup_hdm(port, info);
else
cxlhdm = devm_cxl_setup_hdm(port, info);
@@ -149,27 +149,12 @@ struct cxl_hdm *__wrap_devm_cxl_setup_hdm(struct cxl_port *port,
}
EXPORT_SYMBOL_NS_GPL(__wrap_devm_cxl_setup_hdm, CXL);
-int __wrap_devm_cxl_enable_hdm(struct cxl_port *port, struct cxl_hdm *cxlhdm)
-{
- int index, rc;
- struct cxl_mock_ops *ops = get_cxl_mock_ops(&index);
-
- if (ops && ops->is_mock_port(port->uport))
- rc = 0;
- else
- rc = devm_cxl_enable_hdm(port, cxlhdm);
- put_cxl_mock_ops(index);
-
- return rc;
-}
-EXPORT_SYMBOL_NS_GPL(__wrap_devm_cxl_enable_hdm, CXL);
-
int __wrap_devm_cxl_add_passthrough_decoder(struct cxl_port *port)
{
int rc, index;
struct cxl_mock_ops *ops = get_cxl_mock_ops(&index);
- if (ops && ops->is_mock_port(port->uport))
+ if (ops && ops->is_mock_port(port->uport_dev))
rc = ops->devm_cxl_add_passthrough_decoder(port);
else
rc = devm_cxl_add_passthrough_decoder(port);
@@ -186,7 +171,7 @@ int __wrap_devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm,
struct cxl_port *port = cxlhdm->port;
struct cxl_mock_ops *ops = get_cxl_mock_ops(&index);
- if (ops && ops->is_mock_port(port->uport))
+ if (ops && ops->is_mock_port(port->uport_dev))
rc = ops->devm_cxl_enumerate_decoders(cxlhdm, info);
else
rc = devm_cxl_enumerate_decoders(cxlhdm, info);
@@ -201,7 +186,7 @@ int __wrap_devm_cxl_port_enumerate_dports(struct cxl_port *port)
int rc, index;
struct cxl_mock_ops *ops = get_cxl_mock_ops(&index);
- if (ops && ops->is_mock_port(port->uport))
+ if (ops && ops->is_mock_port(port->uport_dev))
rc = ops->devm_cxl_port_enumerate_dports(port);
else
rc = devm_cxl_port_enumerate_dports(port);
@@ -259,24 +244,46 @@ int __wrap_cxl_dvsec_rr_decode(struct device *dev, int dvsec,
}
EXPORT_SYMBOL_NS_GPL(__wrap_cxl_dvsec_rr_decode, CXL);
-resource_size_t __wrap_cxl_rcrb_to_component(struct device *dev,
- resource_size_t rcrb,
- enum cxl_rcrb which)
+struct cxl_dport *__wrap_devm_cxl_add_rch_dport(struct cxl_port *port,
+ struct device *dport_dev,
+ int port_id,
+ resource_size_t rcrb)
+{
+ int index;
+ struct cxl_dport *dport;
+ struct cxl_mock_ops *ops = get_cxl_mock_ops(&index);
+
+ if (ops && ops->is_mock_port(dport_dev)) {
+ dport = devm_cxl_add_dport(port, dport_dev, port_id,
+ CXL_RESOURCE_NONE);
+ if (!IS_ERR(dport)) {
+ dport->rcrb.base = rcrb;
+ dport->rch = true;
+ }
+ } else
+ dport = devm_cxl_add_rch_dport(port, dport_dev, port_id, rcrb);
+ put_cxl_mock_ops(index);
+
+ return dport;
+}
+EXPORT_SYMBOL_NS_GPL(__wrap_devm_cxl_add_rch_dport, CXL);
+
+resource_size_t __wrap_cxl_rcd_component_reg_phys(struct device *dev,
+ struct cxl_dport *dport)
{
int index;
resource_size_t component_reg_phys;
struct cxl_mock_ops *ops = get_cxl_mock_ops(&index);
if (ops && ops->is_mock_port(dev))
- component_reg_phys =
- ops->cxl_rcrb_to_component(dev, rcrb, which);
+ component_reg_phys = CXL_RESOURCE_NONE;
else
- component_reg_phys = cxl_rcrb_to_component(dev, rcrb, which);
+ component_reg_phys = cxl_rcd_component_reg_phys(dev, dport);
put_cxl_mock_ops(index);
return component_reg_phys;
}
-EXPORT_SYMBOL_NS_GPL(__wrap_cxl_rcrb_to_component, CXL);
+EXPORT_SYMBOL_NS_GPL(__wrap_cxl_rcd_component_reg_phys, CXL);
MODULE_LICENSE("GPL v2");
MODULE_IMPORT_NS(ACPI);
diff --git a/tools/testing/cxl/test/mock.h b/tools/testing/cxl/test/mock.h
index bef8817..a942237 100644
--- a/tools/testing/cxl/test/mock.h
+++ b/tools/testing/cxl/test/mock.h
@@ -15,9 +15,6 @@ struct cxl_mock_ops {
acpi_string pathname,
struct acpi_object_list *arguments,
unsigned long long *data);
- resource_size_t (*cxl_rcrb_to_component)(struct device *dev,
- resource_size_t rcrb,
- enum cxl_rcrb which);
struct acpi_pci_root *(*acpi_pci_find_root)(acpi_handle handle);
bool (*is_mock_bus)(struct pci_bus *bus);
bool (*is_mock_port)(struct device *dev);
diff --git a/tools/testing/nvdimm/test/nfit.c b/tools/testing/nvdimm/test/nfit.c
index e4e2d16..005043b 100644
--- a/tools/testing/nvdimm/test/nfit.c
+++ b/tools/testing/nvdimm/test/nfit.c
@@ -3240,11 +3240,6 @@ static int nfit_test_probe(struct platform_device *pdev)
return 0;
}
-static int nfit_test_remove(struct platform_device *pdev)
-{
- return 0;
-}
-
static void nfit_test_release(struct device *dev)
{
struct nfit_test *nfit_test = to_nfit_test(dev);
@@ -3259,7 +3254,6 @@ static const struct platform_device_id nfit_test_id[] = {
static struct platform_driver nfit_test_driver = {
.probe = nfit_test_probe,
- .remove = nfit_test_remove,
.driver = {
.name = KBUILD_MODNAME,
},
diff --git a/tools/testing/nvdimm/test/nfit_test.h b/tools/testing/nvdimm/test/nfit_test.h
index b5f7a99..b00583d 100644
--- a/tools/testing/nvdimm/test/nfit_test.h
+++ b/tools/testing/nvdimm/test/nfit_test.h
@@ -207,7 +207,36 @@ typedef struct nfit_test_resource *(*nfit_test_lookup_fn)(resource_size_t);
typedef union acpi_object *(*nfit_test_evaluate_dsm_fn)(acpi_handle handle,
const guid_t *guid, u64 rev, u64 func,
union acpi_object *argv4);
+void __iomem *__wrap_devm_ioremap(struct device *dev,
+ resource_size_t offset, unsigned long size);
+void *__wrap_devm_memremap(struct device *dev, resource_size_t offset,
+ size_t size, unsigned long flags);
+void *__wrap_devm_memremap_pages(struct device *dev, struct dev_pagemap *pgmap);
+pfn_t __wrap_phys_to_pfn_t(phys_addr_t addr, unsigned long flags);
+void *__wrap_memremap(resource_size_t offset, size_t size,
+ unsigned long flags);
+void __wrap_devm_memunmap(struct device *dev, void *addr);
+void __iomem *__wrap_ioremap(resource_size_t offset, unsigned long size);
+void __iomem *__wrap_ioremap_wc(resource_size_t offset, unsigned long size);
void __wrap_iounmap(volatile void __iomem *addr);
+void __wrap_memunmap(void *addr);
+struct resource *__wrap___request_region(struct resource *parent,
+ resource_size_t start, resource_size_t n, const char *name,
+ int flags);
+int __wrap_insert_resource(struct resource *parent, struct resource *res);
+int __wrap_remove_resource(struct resource *res);
+struct resource *__wrap___devm_request_region(struct device *dev,
+ struct resource *parent, resource_size_t start,
+ resource_size_t n, const char *name);
+void __wrap___release_region(struct resource *parent, resource_size_t start,
+ resource_size_t n);
+void __wrap___devm_release_region(struct device *dev, struct resource *parent,
+ resource_size_t start, resource_size_t n);
+acpi_status __wrap_acpi_evaluate_object(acpi_handle handle, acpi_string path,
+ struct acpi_object_list *p, struct acpi_buffer *buf);
+union acpi_object * __wrap_acpi_evaluate_dsm(acpi_handle handle, const guid_t *guid,
+ u64 rev, u64 func, union acpi_object *argv4);
+
void nfit_test_setup(nfit_test_lookup_fn lookup,
nfit_test_evaluate_dsm_fn evaluate);
void nfit_test_teardown(void);
