blkdev: Refactoring block io latency histogram codes
The current io_latency_state structure includes entries for read
and write requests. There are special types of write commands
such as sync and discard (trim) commands, and the current
implementation is not general enough if we want to separate
latency histogram for such special commands.
This change makes io_latency_state structure request-type neutral.
It also changes to print the latency average.
Signed-off-by: Hyojun Kim <hyojun@google.com>
diff --git a/block/blk-core.c b/block/blk-core.c
index d178877..aac0184 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -3574,76 +3574,43 @@
* TODO : If necessary, we can make the histograms per-cpu and aggregate
* them when printing them out.
*/
-void
-blk_zero_latency_hist(struct io_latency_state *s)
-{
- memset(s->latency_y_axis_read, 0,
- sizeof(s->latency_y_axis_read));
- memset(s->latency_y_axis_write, 0,
- sizeof(s->latency_y_axis_write));
- s->latency_reads_elems = 0;
- s->latency_writes_elems = 0;
-}
-EXPORT_SYMBOL(blk_zero_latency_hist);
-
ssize_t
-blk_latency_hist_show(struct io_latency_state *s, char *buf)
+blk_latency_hist_show(char* name, struct io_latency_state *s, char *buf,
+ int buf_size)
{
int i;
int bytes_written = 0;
u_int64_t num_elem, elem;
int pct;
+ u_int64_t average;
- num_elem = s->latency_reads_elems;
- if (num_elem > 0) {
- bytes_written += scnprintf(buf + bytes_written,
- PAGE_SIZE - bytes_written,
- "IO svc_time Read Latency Histogram (n = %llu):\n",
- num_elem);
- for (i = 0;
- i < ARRAY_SIZE(latency_x_axis_us);
- i++) {
- elem = s->latency_y_axis_read[i];
- pct = div64_u64(elem * 100, num_elem);
- bytes_written += scnprintf(buf + bytes_written,
- PAGE_SIZE - bytes_written,
- "\t< %5lluus%15llu%15d%%\n",
- latency_x_axis_us[i],
- elem, pct);
- }
- /* Last element in y-axis table is overflow */
- elem = s->latency_y_axis_read[i];
- pct = div64_u64(elem * 100, num_elem);
- bytes_written += scnprintf(buf + bytes_written,
- PAGE_SIZE - bytes_written,
- "\t> %5dms%15llu%15d%%\n", 10,
- elem, pct);
+ num_elem = s->latency_elems;
+ if (num_elem > 0) {
+ average = div64_u64(s->latency_sum, s->latency_elems);
+ bytes_written += scnprintf(buf + bytes_written,
+ buf_size - bytes_written,
+ "IO svc_time %s Latency Histogram (n = %llu,"
+ " average = %llu):\n", name, num_elem, average);
+ for (i = 0;
+ i < ARRAY_SIZE(latency_x_axis_us);
+ i++) {
+ elem = s->latency_y_axis[i];
+ pct = div64_u64(elem * 100, num_elem);
+ bytes_written += scnprintf(buf + bytes_written,
+ PAGE_SIZE - bytes_written,
+ "\t< %6lluus%15llu%15d%%\n",
+ latency_x_axis_us[i],
+ elem, pct);
+ }
+ /* Last element in y-axis table is overflow */
+ elem = s->latency_y_axis[i];
+ pct = div64_u64(elem * 100, num_elem);
+ bytes_written += scnprintf(buf + bytes_written,
+ PAGE_SIZE - bytes_written,
+ "\t>=%6lluus%15llu%15d%%\n",
+ latency_x_axis_us[i - 1], elem, pct);
}
- num_elem = s->latency_writes_elems;
- if (num_elem > 0) {
- bytes_written += scnprintf(buf + bytes_written,
- PAGE_SIZE - bytes_written,
- "IO svc_time Write Latency Histogram (n = %llu):\n",
- num_elem);
- for (i = 0;
- i < ARRAY_SIZE(latency_x_axis_us);
- i++) {
- elem = s->latency_y_axis_write[i];
- pct = div64_u64(elem * 100, num_elem);
- bytes_written += scnprintf(buf + bytes_written,
- PAGE_SIZE - bytes_written,
- "\t< %5lluus%15llu%15d%%\n",
- latency_x_axis_us[i],
- elem, pct);
- }
- /* Last element in y-axis table is overflow */
- elem = s->latency_y_axis_write[i];
- pct = div64_u64(elem * 100, num_elem);
- bytes_written += scnprintf(buf + bytes_written,
- PAGE_SIZE - bytes_written,
- "\t> %5dms%15llu%15d%%\n", 10,
- elem, pct);
- }
+
return bytes_written;
}
EXPORT_SYMBOL(blk_latency_hist_show);
diff --git a/drivers/mmc/core/core.c b/drivers/mmc/core/core.c
index 2986e27..ea3a727 100644
--- a/drivers/mmc/core/core.c
+++ b/drivers/mmc/core/core.c
@@ -191,9 +191,10 @@
completion = ktime_get();
delta_us = ktime_us_delta(completion,
mrq->io_start);
- blk_update_latency_hist(&host->io_lat_s,
- (mrq->data->flags & MMC_DATA_READ),
- delta_us);
+ blk_update_latency_hist(
+ (mrq->data->flags & MMC_DATA_READ) ?
+ &host->io_lat_read :
+ &host->io_lat_write, delta_us);
}
#endif
trace_mmc_blk_rw_end(cmd->opcode, cmd->arg, mrq->data);
@@ -2932,8 +2933,14 @@
latency_hist_show(struct device *dev, struct device_attribute *attr, char *buf)
{
struct mmc_host *host = cls_dev_to_mmc_host(dev);
+ size_t written_bytes;
- return blk_latency_hist_show(&host->io_lat_s, buf);
+ written_bytes = blk_latency_hist_show("Read", &host->io_lat_read,
+ buf, PAGE_SIZE);
+ written_bytes += blk_latency_hist_show("Write", &host->io_lat_write,
+ buf + written_bytes, PAGE_SIZE - written_bytes);
+
+ return written_bytes;
}
/*
@@ -2951,9 +2958,10 @@
if (kstrtol(buf, 0, &value))
return -EINVAL;
- if (value == BLK_IO_LAT_HIST_ZERO)
- blk_zero_latency_hist(&host->io_lat_s);
- else if (value == BLK_IO_LAT_HIST_ENABLE ||
+ if (value == BLK_IO_LAT_HIST_ZERO) {
+ memset(&host->io_lat_read, 0, sizeof(host->io_lat_read));
+ memset(&host->io_lat_write, 0, sizeof(host->io_lat_write));
+ } else if (value == BLK_IO_LAT_HIST_ENABLE ||
value == BLK_IO_LAT_HIST_DISABLE)
host->latency_hist_enabled = value;
return count;
diff --git a/drivers/scsi/ufs/ufshcd.c b/drivers/scsi/ufs/ufshcd.c
index 1b9008ca..4bc2ee1 100644
--- a/drivers/scsi/ufs/ufshcd.c
+++ b/drivers/scsi/ufs/ufshcd.c
@@ -3207,10 +3207,10 @@
completion = ktime_get();
delta_us = ktime_us_delta(completion,
req->lat_hist_io_start);
- /* rq_data_dir() => true if WRITE */
- blk_update_latency_hist(&hba->io_lat_s,
- (rq_data_dir(req) == READ),
- delta_us);
+ blk_update_latency_hist(
+ (rq_data_dir(req) == READ) ?
+ &hba->io_lat_read :
+ &hba->io_lat_write, delta_us);
}
}
/* Do not touch lrbp after scsi done */
@@ -5382,9 +5382,10 @@
if (kstrtol(buf, 0, &value))
return -EINVAL;
- if (value == BLK_IO_LAT_HIST_ZERO)
- blk_zero_latency_hist(&hba->io_lat_s);
- else if (value == BLK_IO_LAT_HIST_ENABLE ||
+ if (value == BLK_IO_LAT_HIST_ZERO) {
+ memset(&hba->io_lat_read, 0, sizeof(hba->io_lat_read));
+ memset(&hba->io_lat_write, 0, sizeof(hba->io_lat_write));
+ } else if (value == BLK_IO_LAT_HIST_ENABLE ||
value == BLK_IO_LAT_HIST_DISABLE)
hba->latency_hist_enabled = value;
return count;
@@ -5395,8 +5396,14 @@
char *buf)
{
struct ufs_hba *hba = dev_get_drvdata(dev);
+ size_t written_bytes;
- return blk_latency_hist_show(&hba->io_lat_s, buf);
+ written_bytes = blk_latency_hist_show("Read", &hba->io_lat_read,
+ buf, PAGE_SIZE);
+ written_bytes += blk_latency_hist_show("Write", &hba->io_lat_write,
+ buf + written_bytes, PAGE_SIZE - written_bytes);
+
+ return written_bytes;
}
static DEVICE_ATTR(latency_hist, S_IRUGO | S_IWUSR,
diff --git a/drivers/scsi/ufs/ufshcd.h b/drivers/scsi/ufs/ufshcd.h
index b74eedb..c1310ead 100644
--- a/drivers/scsi/ufs/ufshcd.h
+++ b/drivers/scsi/ufs/ufshcd.h
@@ -542,7 +542,8 @@
bool is_sys_suspended;
int latency_hist_enabled;
- struct io_latency_state io_lat_s;
+ struct io_latency_state io_lat_read;
+ struct io_latency_state io_lat_write;
};
/* Returns true if clocks can be gated. Otherwise false */
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index a9562bb..fb5302d 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -1694,43 +1694,26 @@
#define BLK_IO_LAT_HIST_ZERO 2
struct io_latency_state {
- u_int64_t latency_y_axis_read[ARRAY_SIZE(latency_x_axis_us) + 1];
- u_int64_t latency_reads_elems;
- u_int64_t latency_y_axis_write[ARRAY_SIZE(latency_x_axis_us) + 1];
- u_int64_t latency_writes_elems;
+ u_int64_t latency_y_axis[ARRAY_SIZE(latency_x_axis_us) + 1];
+ u_int64_t latency_elems;
+ u_int64_t latency_sum;
};
static inline void
-blk_update_latency_hist(struct io_latency_state *s,
- int read,
- u_int64_t delta_us)
+blk_update_latency_hist(struct io_latency_state *s, u_int64_t delta_us)
{
int i;
- for (i = 0; i < ARRAY_SIZE(latency_x_axis_us); i++) {
- if (delta_us < (u_int64_t)latency_x_axis_us[i]) {
- if (read)
- s->latency_y_axis_read[i]++;
- else
- s->latency_y_axis_write[i]++;
+ for (i = 0; i < ARRAY_SIZE(latency_x_axis_us); i++)
+ if (delta_us < (u_int64_t)latency_x_axis_us[i])
break;
- }
- }
- if (i == ARRAY_SIZE(latency_x_axis_us)) {
- /* Overflowed the histogram */
- if (read)
- s->latency_y_axis_read[i]++;
- else
- s->latency_y_axis_write[i]++;
- }
- if (read)
- s->latency_reads_elems++;
- else
- s->latency_writes_elems++;
+ s->latency_y_axis[i]++;
+ s->latency_elems++;
+ s->latency_sum += delta_us;
}
-void blk_zero_latency_hist(struct io_latency_state *s);
-ssize_t blk_latency_hist_show(struct io_latency_state *s, char *buf);
+ssize_t blk_latency_hist_show(char* name, struct io_latency_state *s,
+ char *buf, int buf_size);
#else /* CONFIG_BLOCK */
diff --git a/include/linux/mmc/host.h b/include/linux/mmc/host.h
index 97b2b0b..46e91d4 100644
--- a/include/linux/mmc/host.h
+++ b/include/linux/mmc/host.h
@@ -382,7 +382,8 @@
#ifdef CONFIG_BLOCK
int latency_hist_enabled;
- struct io_latency_state io_lat_s;
+ struct io_latency_state io_lat_read;
+ struct io_latency_state io_lat_write;
#endif
unsigned long private[0] ____cacheline_aligned;
