Snap for 12359238 from 68a2b1e376d177bf3773ac8a2f17630e61c0c284 to android15-tests-release
Change-Id: Ia502b8d1473fb263a1332c64bd66b0364f649259
diff --git a/app/memorylatencybench/main.c b/app/memorylatencybench/main.c
index 7139896..a6015ac 100644
--- a/app/memorylatencybench/main.c
+++ b/app/memorylatencybench/main.c
@@ -81,7 +81,11 @@
}
}
+static uint64_t mem_lat_pmu_evt_arr[] = {PMU_EV_CPU_CYCLES,
+ PMU_EV_INST_RETIRED};
+
BENCH_SETUP(memlatency) {
+ BENCH_INIT_PMU(mem_lat_pmu_evt_arr);
trusty_bench_get_param_name_cb = &get_param_name_cb_fixed;
trusty_bench_get_formatted_value_cb = &get_formatted_value_cb;
memlatency_state_start =
@@ -163,4 +167,16 @@
return bench_get_duration_ns();
}
+BENCH_RESULT(memlatency, latency_read, cycle_counter_0) {
+ return bench_get_pmu_cnt(0);
+}
+
+BENCH_RESULT(memlatency, latency_read, cycle_counter) {
+ return bench_get_pmu_cnt(1);
+}
+
+BENCH_RESULT(memlatency, latency_read, inst_retired) {
+ return bench_get_pmu_cnt(2);
+}
+
PORT_TEST(memlatency, "com.android.kernel.memorylatency.bench");
diff --git a/app/trusty/user-tasks.mk b/app/trusty/user-tasks.mk
index abbad19..535cf64 100644
--- a/app/trusty/user-tasks.mk
+++ b/app/trusty/user-tasks.mk
@@ -109,10 +109,13 @@
# account for differing licenses across projects.
TRUSTY_SDK_MODULES := \
external/boringssl \
+ external/open-dice \
trusty/kernel/lib/libc-ext \
trusty/kernel/lib/ubsan \
trusty/user/base/interface/hwaes \
+ trusty/user/base/interface/hwbcc \
trusty/user/base/interface/hwkey \
+ trusty/user/base/interface/keybox \
trusty/user/base/interface/keymaster \
trusty/user/base/interface/spi \
trusty/user/base/interface/storage \
@@ -120,9 +123,11 @@
trusty/user/base/lib/dlmalloc \
trusty/user/base/lib/googletest \
trusty/user/base/lib/hwaes \
+ trusty/user/base/lib/hwbcc/client \
trusty/user/base/lib/hwbcc/rust \
trusty/user/base/lib/hwkey \
trusty/user/base/lib/hwkey/rust \
+ trusty/user/base/lib/keybox/client \
trusty/user/base/lib/keymaster \
trusty/user/base/lib/libc-trusty \
trusty/user/base/lib/libcxxabi-trusty \
diff --git a/include/shared/lk/trusty_bench_common.h b/include/shared/lk/trusty_bench_common.h
index 629392e..0b55c5d 100644
--- a/include/shared/lk/trusty_bench_common.h
+++ b/include/shared/lk/trusty_bench_common.h
@@ -94,6 +94,7 @@
int64_t (*bench_result)(void);
trusty_bench_get_formatted_value_callback_t formatted_value_cb;
trusty_bench_get_param_name_callback_t param_name_cb;
+ trusty_bench_check_results_callback_t check_results_cb;
};
/*
@@ -182,15 +183,20 @@
* @metric_name: Name of the metric for which this value is to be
* formatted
*/
-static inline void trusty_bench_sprint_col_stat(char* buffer,
- size_t buffer_len,
- int64_t val,
- const char* metric_name) {
- if (trusty_bench_get_formatted_value_cb == NULL) {
+static inline void trusty_bench_sprint_col_stat(
+ char* buffer,
+ size_t buffer_len,
+ int64_t val,
+ const char* metric_name,
+ trusty_bench_get_formatted_value_callback_t value_format_cb) {
+ if (value_format_cb == NULL) {
+ value_format_cb = trusty_bench_get_formatted_value_cb;
+ }
+
+ if (value_format_cb == NULL) {
snprintf(buffer, buffer_len, "%" PRId64, val);
} else {
- trusty_bench_get_formatted_value_cb(buffer, buffer_len, val,
- metric_name);
+ value_format_cb(buffer, buffer_len, val, metric_name);
EXPECT_EQ(trusty_bench_validate_numeric(buffer), true,
"%s is not a valid double representation.\n", buffer);
}
diff --git a/include/shared/lk/trusty_bench_json_print.h b/include/shared/lk/trusty_bench_json_print.h
index e2636ba..58e9214 100644
--- a/include/shared/lk/trusty_bench_json_print.h
+++ b/include/shared/lk/trusty_bench_json_print.h
@@ -81,18 +81,18 @@
/* print formatted values */
trusty_bench_sprint_col_stat(
buf, sizeof(buf), entry->metric.aggregates[BENCH_AGGREGATE_MIN],
- entry->name);
+ entry->name, entry->formatted_value_cb);
trusty_unittest_printf("\"min\": \"%s\",", buf);
trusty_bench_sprint_col_stat(
buf, sizeof(buf), entry->metric.aggregates[BENCH_AGGREGATE_MAX],
- entry->name);
+ entry->name, entry->formatted_value_cb);
trusty_unittest_printf("\"max\": \"%s\",", buf);
trusty_bench_sprint_col_stat(
buf, sizeof(buf), entry->metric.aggregates[BENCH_AGGREGATE_AVG],
- entry->name);
+ entry->name, entry->formatted_value_cb);
trusty_unittest_printf("\"avg\": \"%s\",", buf);
trusty_bench_sprint_col_stat(buf, sizeof(buf), entry->metric.cold,
- entry->name);
+ entry->name, entry->formatted_value_cb);
trusty_unittest_printf("\"cold\": \"%s\",", buf);
/* Formatting is conditional to Metric Name, so we always print raw
diff --git a/include/shared/lk/trusty_bench_option_cb.h b/include/shared/lk/trusty_bench_option_cb.h
index fa350a0..33697ba 100644
--- a/include/shared/lk/trusty_bench_option_cb.h
+++ b/include/shared/lk/trusty_bench_option_cb.h
@@ -51,6 +51,17 @@
*/
static trusty_bench_get_param_name_callback_t trusty_bench_get_param_name_cb;
+// Forward declaration for the trusty_bench_check_results_callback_t definition
+struct bench_metric_list_node;
+
+/**
+ * typedef trusty_bench_check_results_callback_t - Type of the callback to
+ * check if aggregate values are in desired range
+ * @metric_node: The metric node to be validated
+ */
+typedef bool (*trusty_bench_check_results_callback_t)(
+ struct bench_metric_list_node* metric_node);
+
/**
* typedef trusty_bench_get_formatted_value_callback - Type of the callback to
* customize value printing
diff --git a/include/shared/lk/trusty_bench_print_tables.h b/include/shared/lk/trusty_bench_print_tables.h
index 0c6c944..97eef58 100644
--- a/include/shared/lk/trusty_bench_print_tables.h
+++ b/include/shared/lk/trusty_bench_print_tables.h
@@ -212,7 +212,7 @@
/* Get the size of the max value */
trusty_bench_sprint_col_stat(
buf, sizeof(buf), entry->metric.aggregates[BENCH_AGGREGATE_MAX],
- entry->name);
+ entry->name, entry->formatted_value_cb);
trusty_bench_max_column_width =
MAX(strnlen(buf, sizeof(buf)), trusty_bench_max_column_width);
trusty_bench_max_metric_digit_width = MAX(
@@ -221,11 +221,12 @@
/* Get the size of the min value, because aggregates are signed */
trusty_bench_sprint_col_stat(
buf, sizeof(buf), entry->metric.aggregates[BENCH_AGGREGATE_MIN],
- entry->name);
+ entry->name, entry->formatted_value_cb);
trusty_bench_max_column_width =
MAX(strnlen(buf, sizeof(buf)), trusty_bench_max_column_width);
trusty_bench_max_metric_digit_width = MAX(
trusty_bench_max_metric_digit_width, strnlen(buf, sizeof(buf)));
+ column_width = MAX(column_width, trusty_bench_max_metric_digit_width);
/* Check Column is not too big */
if (trusty_bench_max_column_width > BENCH_MAX_COL_SIZE) {
diff --git a/include/shared/lk/trusty_benchmark.h b/include/shared/lk/trusty_benchmark.h
index 1bb4728..8a1019c 100644
--- a/include/shared/lk/trusty_benchmark.h
+++ b/include/shared/lk/trusty_benchmark.h
@@ -125,6 +125,7 @@
#include <stdarg.h>
#include <stdlib.h>
+#include <lib/pmu/pmu_arch.h>
#include <lib/unittest/unittest.h>
#include <trusty_log.h>
#include "trusty_bench_common.h"
@@ -141,6 +142,10 @@
#endif
#include <uapi/err.h>
+#ifdef WITH_TEST_PMU
+#include <lib/pmu/pmu.h>
+#endif
+
/*
* A few helper macros for static dispatch
*/
@@ -159,10 +164,14 @@
* @last_bench_body_duration: nanoseconds duration of the last execution of
* the bench body.
* @cur_param_idx: index of current parameter in param_array.
+ * @pmu: state of pmu counters
*/
static struct benchmark_internal_state {
int64_t last_bench_body_duration;
size_t cur_param_idx;
+#ifdef WITH_TEST_PMU
+ struct trusty_pmu_state pmu;
+#endif
} bench_state;
/**
@@ -300,6 +309,29 @@
}
/**
+ * trusty_bench_check_metrics - Check if All Metric are within range
+ * after one iteration of bench function for all param/metric in the last BENCH.
+ * @metric_list: List of metrics aggregated during all BENCH runs.
+ * @param_idx: Index of the current parameter in the param_array of
+ * BENCH.
+ */
+static inline bool trusty_bench_check_metrics(struct list_node* metric_list,
+ size_t param_idx) {
+ struct bench_metric_list_node* entry;
+
+ list_for_every_entry(metric_list, entry, struct bench_metric_list_node,
+ node) {
+ if (param_idx == entry->param_idx) {
+ if (entry->check_results_cb != NULL &&
+ !entry->check_results_cb(entry)) {
+ return false;
+ }
+ }
+ }
+ return true;
+}
+
+/**
* trusty_bench_reset_metrics - Run All Metric Updaters after one
* iteration of bench function for all param/metric in the last BENCH.
* @metric_list: List of metrics aggregated during all BENCH runs.
@@ -344,14 +376,20 @@
* @suite_name: Identifier of the current suite.
* @bench_name: Unique identifier of the Bench in the suite.
* @metric_name: Name of the metric to print in the result table.
- * @formatted_value_cb: [optional] A callback of
- * trusty_bench_get_formatted_value_callback_t type for formatting the result
- * value to a string
- * @param_name_cb: [optional] A callback of
- * trusty_bench_get_param_name_callback_t type for formatting the param name
+ * @formatted_value_cb: [optional] A callback of
+ * trusty_bench_get_formatted_value_callback_t type
+ * for formatting the result
+ * value to a string
+ * @param_name_cb: [optional] A callback of
+ * trusty_bench_get_param_name_callback_t type
+ * for formatting the param name
+ * @check_results_cb: [optional] A callback of
+ * trusty_bench_check_results_callback_t
+ * type for formatting the param name
*/
#define BENCH_RESULT_INNER(suite_name, bench_name, metric_name, \
- formatted_value_cb_, param_name_cb_) \
+ formatted_value_cb_, param_name_cb_, \
+ check_results_cb_) \
static int64_t update_##suite_name##_##bench_name##_##metric_name(void); \
static struct bench_metric_list_node \
suite_name##_##bench_name##_##metric_name##_node = { \
@@ -363,7 +401,8 @@
.bench_result = \
update_##suite_name##_##bench_name##_##metric_name, \
.formatted_value_cb = formatted_value_cb_, \
- .param_name_cb = param_name_cb_}; \
+ .param_name_cb = param_name_cb_, \
+ .check_results_cb = check_results_cb_}; \
__attribute__((constructor)) void \
suite_name##_##bench_name##_##metric_name##_add(void) { \
list_add_tail(&suite_name##_##bench_name##_metric_list, \
@@ -374,16 +413,38 @@
/* Dispatch Mechanics for BENCH_RESULT */
#define BENCH_RESULT_3(suite_name, bench_name, metric_name) \
- BENCH_RESULT_INNER(suite_name, bench_name, metric_name, 0, 0)
+ BENCH_RESULT_INNER(suite_name, bench_name, metric_name, 0, 0, 0)
+
#define BENCH_RESULT_4(suite_name, bench_name, metric_name, \
formatted_value_cb) \
BENCH_RESULT_INNER(suite_name, bench_name, metric_name, \
- formatted_value_cb, 0)
+ formatted_value_cb, 0, 0)
+
#define BENCH_RESULT_5(suite_name, bench_name, metric_name, \
formatted_value_cb, param_name_cb) \
BENCH_RESULT_INNER(suite_name, bench_name, metric_name, \
- formatted_value_cb, param_name_cb)
+ formatted_value_cb, param_name_cb, 0)
+#define BENCH_RESULT_6(suite_name, bench_name, metric_name, \
+ formatted_value_cb, param_name_cb, check_results_cb) \
+ BENCH_RESULT_INNER(suite_name, bench_name, metric_name, \
+ formatted_value_cb, param_name_cb, check_results_cb)
+
+#ifdef WITH_TEST_PMU
+/**
+ * bench_get_pmu_cnt - convenience function to use in BENCH_RESULT to get
+ * the value of a pmu counter for the last bench body execution.
+ *
+ * Return: The value of a pmu counter of the last completed BENCH body.
+ */
+static inline int64_t bench_get_pmu_cnt(size_t idx) {
+ return bench_state.pmu.vals[idx];
+}
+#else
+static inline int64_t bench_get_pmu_cnt(size_t idx) {
+ return 0;
+}
+#endif
/**
* BENCH_RESULT Dispatch BENCH_RESULT Called 3, 4 or 5 parameters.
* @suite_name: Identifier of the current suite.
@@ -468,6 +529,14 @@
TLOGE("Failed to Allocate memory for bench_metric_list_node!");
return NULL;
}
+
+ // clear parameterized_list from previous runs
+ struct list_node* node = NULL;
+ do {
+ node = list_remove_head(parameterized_list);
+ free(node);
+ } while (node != NULL);
+
list_for_every_entry(unparameterized_list, entry,
struct bench_metric_list_node, node) {
for (size_t idx_param = 0; idx_param < nb_params; ++idx_param) {
@@ -480,6 +549,7 @@
list_pool[idx].bench_result = entry->bench_result;
list_pool[idx].formatted_value_cb = entry->formatted_value_cb;
list_pool[idx].param_name_cb = entry->param_name_cb;
+ list_pool[idx].check_results_cb = entry->check_results_cb;
list_add_tail(parameterized_list, &(list_pool[idx].node));
++idx;
}
@@ -538,6 +608,21 @@
return 0;
}
+#ifdef WITH_TEST_PMU
+#define BENCH_INIT_PMU(evt_arr) \
+ init_pmu_state(evt_arr, countof(evt_arr), &bench_state.pmu)
+#define PMU_START() pmu_start(&bench_state.pmu);
+#define PMU_STOP() pmu_stop(&bench_state.pmu);
+#define RESET_PMU() reset_pmu_cnts(&bench_state.pmu)
+#define CLEAN_PMU() clean_pmu(&bench_state.pmu)
+#else
+#define BENCH_INIT_PMU(evt_arr) (void)(evt_arr)
+#define PMU_START()
+#define PMU_STOP()
+#define RESET_PMU()
+#define CLEAN_PMU()
+#endif
+
/**
* BENCH_CORE - Called by both parametrized and unparameterized
* BENCH for their common part
@@ -589,6 +674,7 @@
} \
int64_t overhead = trusty_bench_get_overhead(); \
\
+ PMU_START(); \
/* Cold Run */ \
int64_t start_time; \
int64_t end_time; \
@@ -596,6 +682,8 @@
int64_t res = suite_name##_##bench_name##_inner_##params(); \
end_time = get_current_time_ns(); \
\
+ PMU_STOP(); \
+ \
if (res != NO_ERROR) { \
TLOGE("ERROR During Cold Run%" PRId64 "\n", res); \
_test_context.all_ok = false; \
@@ -615,13 +703,17 @@
\
if (!_test_context.hard_fail && _test_context.all_ok) { \
trusty_bench_run_metrics(&metric_list, idx_param, true); \
+ RESET_PMU(); \
} \
\
for (size_t idx_run = 0; idx_run < nb_runs; ++idx_run) { \
if (!_test_context.hard_fail && _test_context.all_ok) { \
+ PMU_START(); \
start_time = get_current_time_ns(); \
res = suite_name##_##bench_name##_inner_##params(); \
end_time = get_current_time_ns(); \
+ PMU_STOP(); \
+ \
bench_state.last_bench_body_duration = end_time - start_time; \
if (overhead >= bench_state.last_bench_body_duration) { \
TLOGE("Benchmark internal function is too fast %" PRId64 \
@@ -637,6 +729,7 @@
} \
if (!_test_context.hard_fail && _test_context.all_ok) { \
trusty_bench_run_metrics(&metric_list, idx_param, false); \
+ RESET_PMU(); \
} \
} \
suite_name##_teardown(); \
@@ -644,6 +737,11 @@
if (rc != NO_ERROR) { \
TLOGW("failed to reset CPU affinity: %d\n", rc); \
} \
+ \
+ if (!trusty_bench_check_metrics(&metric_list, idx_param)) { \
+ _test_context.all_ok = false; \
+ _test_context.tests_failed++; \
+ } \
TEST_END_FUNC(); \
free(extended_test_name); \
extended_test_name = NULL; \
@@ -652,7 +750,8 @@
STRINGIFY(suite_name), \
STRINGIFY(bench_name##_##params)); \
trusty_bench_get_param_name_cb = NULL; \
- trusty_bench_get_formatted_value_cb = NULL
+ trusty_bench_get_formatted_value_cb = NULL; \
+ CLEAN_PMU();
/**
* BENCH_PARAMETERIZED_PTR -Called when BENCH has 5 parameters. This allows
diff --git a/lib/pmu/include/lib/pmu/pmu.h b/lib/pmu/include/lib/pmu/pmu.h
new file mode 100644
index 0000000..1a00ece
--- /dev/null
+++ b/lib/pmu/include/lib/pmu/pmu.h
@@ -0,0 +1,431 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files
+ * (the "Software"), to deal in the Software without restriction,
+ * including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#pragma once
+
+#include <inttypes.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include "pmu_arch.h"
+
+#ifndef ARM64_READ_SYSREG
+#define DSB __asm__ volatile("dsb sy" ::: "memory")
+#define ISB __asm__ volatile("isb" ::: "memory")
+
+#define STRINGIFY(x) #x
+#define TOSTRING(x) STRINGIFY(x)
+
+#define ARM64_READ_SYSREG(reg) \
+ ({ \
+ uint64_t _val; \
+ __asm__ volatile("mrs %0," TOSTRING(reg) : "=r"(_val)); \
+ _val; \
+ })
+
+#define ARM64_WRITE_SYSREG_RAW(reg, val) \
+ ({ __asm__ volatile("msr " TOSTRING(reg) ", %0" ::"r"(val)); })
+
+#define ARM64_WRITE_SYSREG(reg, val) \
+ ({ \
+ ARM64_WRITE_SYSREG_RAW(reg, val); \
+ ISB; \
+ })
+#endif
+
+static uint64_t set_bit(uint64_t reg, uint64_t bit_number) {
+ return reg | bit_number;
+}
+
+static uint64_t clear_bit(uint64_t reg, uint64_t bit_number) {
+ return reg & (~bit_number);
+}
+
+/**
+ * get_pmu_feat - Returns ID_DFR0_EL1.PerfMon Bits
+ *
+ * Return: ID_DFR0_EL1.PerfMon Bits
+ */
+static uint64_t get_pmu_feat(void) {
+ uint64_t id_dfr0_el1 = ARM64_READ_SYSREG(ID_DFR0_EL1);
+ id_dfr0_el1 = id_dfr0_el1 >> 24; /* Shift PerfMon field down to bit 0 */
+ id_dfr0_el1 &= 0x7; /* Mask to leave just the PerfMon bits */
+ return id_dfr0_el1;
+}
+
+/**
+ * get_pmn - Returns the number of programmable counters
+ *
+ * Return: The number of available programmable counters
+ */
+static uint64_t get_pmn(void) {
+ uint64_t pmcr_el0 = ARM64_READ_SYSREG(PMCR_EL0);
+ pmcr_el0 = pmcr_el0 >> 11; /* Shift N field down to bit 0 */
+ pmcr_el0 &= 0x1F; /* Mask to leave just the 5 N bits */
+ return pmcr_el0;
+}
+
+/**
+ * pmn_config - Sets the event for a programmable counter to record
+ * @counter: r0 = Which counter to program (e.g. 0 for PMN0, 1 for PMN1)
+ * @event: r1 = The event code (from appropriate TRM or ARM Architecture
+ * Reference Manual)
+ */
+static void pmn_config(uint64_t counter, uint64_t event) {
+ ARM64_WRITE_SYSREG(PMSELR_EL0, counter);
+ ARM64_WRITE_SYSREG(PMXEVTYPER_EL0, event);
+}
+
+/**
+ * set_pmu_filters - Set Event Counter Filters
+ *
+ * @counter: the index of the programmable counter slot to which the filter
+ * applies
+ * @filter_flags: filters to be applied to the programmable counter slot
+ */
+static void set_pmu_filters(uint64_t counter, uint64_t filter_flags) {
+ ARM64_WRITE_SYSREG(PMSELR_EL0, counter);
+ ARM64_WRITE_SYSREG(PMCCFILTR_EL0, filter_flags);
+}
+
+/**
+ * ccnt_divider - Enables/disables the divider (1/64) on CCNT
+ * @divider: r0 = If 0 disable divider, else enable dvider
+ */
+static void ccnt_divider(bool divider) {
+ if (divider) {
+ uint64_t reg = ARM64_READ_SYSREG(PMSELR_EL0);
+ reg = clear_bit(reg, PMCR_EL0_D_BIT);
+ ARM64_WRITE_SYSREG(PMSELR_EL0, reg);
+ } else {
+ uint64_t reg = ARM64_READ_SYSREG(PMSELR_EL0);
+ reg = set_bit(reg, PMCR_EL0_D_BIT);
+ ARM64_WRITE_SYSREG(PMSELR_EL0, reg);
+ }
+}
+
+/**
+ * enable_pmu - Global PMU enable
+ * On ARM11 this enables the PMU, and the counters start immediately
+ * On Cortex this enables the PMU, there are individual enables for the counters
+ */
+static void enable_pmu(void) {
+ uint64_t reg = ARM64_READ_SYSREG(PMCR_EL0);
+ reg = set_bit(reg, PMCR_EL0_E_BIT);
+ ARM64_WRITE_SYSREG(PMCR_EL0, reg);
+}
+
+/**
+ * disable_pmu - Global PMU disable
+ * On Cortex, this overrides the enable state of the individual counters
+ */
+static void disable_pmu(void) {
+ uint64_t reg = ARM64_READ_SYSREG(PMCR_EL0);
+ reg = clear_bit(reg, PMCR_EL0_E_BIT);
+ ARM64_WRITE_SYSREG(PMCR_EL0, reg);
+}
+
+/**
+ * enable_ccnt - Enable the CCNT
+ */
+static void enable_ccnt(void) {
+ uint64_t reg = ARM64_READ_SYSREG(PMCNTENSET_EL0);
+ reg = set_bit(reg, PMCNTENSET_EL0_C_BIT);
+ ARM64_WRITE_SYSREG(PMCNTENSET_EL0, reg);
+}
+
+/**
+ * disable_ccnt - Disable the CCNT
+ */
+static void disable_ccnt(void) {
+ uint64_t reg = ARM64_READ_SYSREG(PMCNTENSET_EL0);
+ reg = clear_bit(reg, PMCNTENSET_EL0_C_BIT);
+ ARM64_WRITE_SYSREG(PMCNTENSET_EL0, reg);
+}
+
+/**
+ * enable_pmn - Enable PMN{n}
+ * @counter: The counter to enable (e.g. 0 for PMN0, 1 for PMN1)
+ */
+static void enable_pmn(uint64_t counter) {
+ uint64_t reg = ARM64_READ_SYSREG(PMCNTENSET_EL0);
+ reg = set_bit(reg, (uint64_t)1 << counter);
+ ARM64_WRITE_SYSREG(PMCNTENSET_EL0, reg);
+}
+
+/**
+ * disable_pmn - Disable PMN{n}
+ * @counter: The counter to disable (e.g. 0 for PMN0, 1 for PMN1)
+ */
+static void disable_pmn(uint64_t counter) {
+ uint64_t reg = ARM64_READ_SYSREG(PMCNTENSET_EL0);
+ reg = clear_bit(reg, (uint64_t)1 << counter);
+ ARM64_WRITE_SYSREG(PMCNTENSET_EL0, reg);
+}
+
+/**
+ * read_ccnt - Returns the value of CCNT
+ *
+ * Return: the value to PMCCNTR_EL0
+ */
+static uint64_t read_ccnt(void) {
+ return ARM64_READ_SYSREG(PMCCNTR_EL0);
+}
+
+/**
+ * read_pmn - Returns the value of PMN{n}
+ * @counter: The counter to read (e.g. 0 for PMN0, 1 for PMN1)
+ *
+ * Return: the value to PMXEVCNTR_EL0 after selecting event
+ */
+static uint64_t read_pmn(uint64_t counter) {
+ counter &= 0x1F;
+ ARM64_WRITE_SYSREG(PMSELR_EL0, counter);
+ return ARM64_READ_SYSREG(PMXEVCNTR_EL0);
+}
+
+/**
+ * read_flags - Returns the value of the overflow flags
+ *
+ * Return: the value of PmovSCLR_EL0
+ */
+static uint64_t read_flags(void) {
+ return ARM64_READ_SYSREG(PmovSCLR_EL0);
+}
+
+/**
+ * write_flags - Writes the overflow flags
+ * @flags: value of the flags to write to PmovSSET_EL0
+ */
+static void write_flags(uint64_t flags) {
+ ARM64_WRITE_SYSREG(PmovSCLR_EL0, flags);
+}
+
+/**
+ * reset_pmn - Resets the programmable counters
+ */
+static void reset_pmn(void) {
+ uint64_t reg = ARM64_READ_SYSREG(PMCR_EL0);
+ reg = set_bit(reg, PMCR_EL0_P_BIT);
+ ARM64_WRITE_SYSREG(PMCR_EL0, reg);
+}
+
+/**
+ * reset_ccnt - Resets the CCNT
+ */
+static void reset_ccnt(void) {
+ uint64_t reg = ARM64_READ_SYSREG(PMCR_EL0);
+ reg = set_bit(reg, PMCR_EL0_C_BIT);
+ ARM64_WRITE_SYSREG(PMCR_EL0, reg);
+}
+/**
+ * get_event_code_string - Get the code name for a given PMU event
+ *
+ * @event: current states of the pmu counters
+ *
+ * Return: The Code of the event as a String
+ */
+static const char* get_event_code_string(int event) {
+ switch (event) {
+ case PMU_EV_SW_INCR:
+ return "SW_INCR";
+ case PMU_EV_L1I_CACHE_REFILL:
+ return "L1I_CACHE_REFILL";
+ case PMU_EV_L1I_TLB_REFILL:
+ return "L1I_TLB_REFILL";
+ case PMU_EV_L1D_CACHE_REFILL:
+ return "L1D_CACHE_REFILL";
+ case PMU_EV_L1D_CACHE:
+ return "L1D_CACHE";
+ case PMU_EV_L1D_TLB_REFILL:
+ return "L1D_TLB_REFILL";
+ case PMU_EV_LD_RETIRED:
+ return "LD_RETIRED";
+ case PMU_EV_ST_RETIRED:
+ return "ST_RETIRED";
+ case PMU_EV_INST_RETIRED:
+ return "INST_RETIRED";
+ case PMU_EV_EXC_TAKEN:
+ return "EXC_TAKEN";
+ case PMU_EV_EXC_RETURN:
+ return "EXC_RETURN";
+ case PMU_EV_CID_WRITE_RETIRED:
+ return "CID_WRITE_RETIRED";
+ case PMU_EV_PC_WRITE_RETIRED:
+ return "PC_WRITE_RETIRED";
+ case PMU_EV_BR_IMMED_RETIRED:
+ return "BR_IMMED_RETIRED";
+ case PMU_EV_UNALIGNED_LDST_RETIRED:
+ return "UNALIGNED_LDST_RETIRED";
+ case PMU_EV_BR_MIS_PRED:
+ return "BR_MIS_PRED";
+ case PMU_EV_CPU_CYCLES:
+ return "CPU_CYCLES";
+ case PMU_EV_BR_PRED:
+ return "BR_PRED";
+ case PMU_EV_MEM_ACCESS:
+ return "MEM_ACCESS";
+ case PMU_EV_L1I_CACHE:
+ return "L1I_CACHE";
+ case PMU_EV_L1D_CACHE_WB:
+ return "L1D_CACHE_WB";
+ case PMU_EV_L2D_CACHE:
+ return "L2D_CACHE";
+ case PMU_EV_L2D_CACHE_REFILL:
+ return "L2D_CACHE_REFILL";
+ case PMU_EV_L2D_CACHE_WB:
+ return "L2D_CACHE_WB";
+ case PMU_EV_BUS_ACCESS:
+ return "BUS_ACCESS";
+ case PMU_EV_MEMORY_ERROR:
+ return "MEMORY_ERROR";
+ case PMU_EV_BUS_CYCLES:
+ return "BUS_CYCLES";
+ case PMU_EV_CHAIN:
+ return "CHAIN";
+ case PMU_EV_BUS_ACCESS_LD:
+ return "BUS_ACCESS_LD";
+ case PMU_EV_BUS_ACCESS_ST:
+ return "BUS_ACCESS_ST";
+ case PMU_EV_BR_INDIRECT_SPEC:
+ return "BR_INDIRECT_SPEC";
+ case PMU_EV_EXC_IRQ:
+ return "EXC_IRQ";
+ case PMU_EV_EXC_FIQ:
+ return "EXC_FIQ";
+ case -1:
+ return "CCNT";
+ default:
+ return "UNKNOWN_EVENT";
+ }
+}
+
+#define RECORD_SEL1 (U(0) | PMCCFILTR_EL0_U_BIT | PMCCFILTR_EL0_NSK_BIT)
+#define RECORD_EL3 \
+ (U(0) | PMCCFILTR_EL0_U_BIT | PMCCFILTR_EL0_P_BIT | PMCCFILTR_EL0_M_BIT)
+#define RECORD_ALL (U(0) | PMCCFILTR_EL0_NSH_BIT)
+
+/**
+ * struct trusty_pmu_state - Holds the current state of pmu slots and counters.
+ * @evts: array of pmu counter events codes with which respective
+ * slot have to be programmed
+ * @vals: array of pmu counter current values
+ * @evt_cnt: number of events/values
+ */
+struct trusty_pmu_state {
+ uint64_t* evts;
+ uint64_t* vals;
+ uint64_t evt_cnt;
+};
+
+/**
+ * pmu_start - Setup events slots and Start recording
+ *
+ * @state: current states of the pmu counters
+ */
+static void pmu_start(struct trusty_pmu_state* state) {
+ if (state == NULL || state->evts == NULL) {
+ return;
+ }
+
+ uint64_t nb_counters = get_pmn();
+
+ if (nb_counters < state->evt_cnt) {
+ fprintf(stderr,
+ "ERROR: There are only %" PRIu64
+ " Programmable Counters, yet you are trying to record %" PRIu64
+ " events.\n",
+ nb_counters, state->evt_cnt);
+
+ return;
+ }
+
+ enable_pmu(); // Enable the PMU
+ reset_ccnt(); // Reset the CCNT (cycle counter)
+ reset_pmn(); // Reset the configurable counters
+ write_flags((1 << 31) | 0xf); // Reset overflow flags
+
+ for (size_t i = 0; i < state->evt_cnt; i++) {
+ pmn_config(i, state->evts[i] | RECORD_ALL);
+ set_pmu_filters(i, 0U | RECORD_ALL);
+ }
+ ccnt_divider(0); // Enable divide by 64
+ enable_ccnt(); // Enable CCNT
+
+ for (size_t i = 0; i < state->evt_cnt; i++)
+ enable_pmn(i);
+}
+
+/**
+ * pmu_stop - Stop recording
+ *
+ * @state: current states of the pmu counters
+ */
+static void pmu_stop(struct trusty_pmu_state* state) {
+ if (state == NULL || state->evts == NULL) {
+ return;
+ }
+ disable_ccnt();
+
+ for (size_t i = 0; i < state->evt_cnt; i++)
+ disable_pmn(i);
+
+ for (size_t i = 0; i < state->evt_cnt; i++) {
+ state->vals[i + 1] = read_pmn(i);
+ }
+ state->vals[0] = read_ccnt(); // Read CCNT
+}
+
+/**
+ * init_pmu_state - Allocate memory for values and bind events to be programmed
+ *
+ * @evt_arr: array of events on whiches to program into the slots
+ * @evt_arr_sz: countof/nb_elements of evt_arr
+ * @pmu_state: current states of the pmu counters
+ */
+static inline void init_pmu_state(uint64_t* evt_arr,
+ size_t evt_arr_sz,
+ struct trusty_pmu_state* pmu_state) {
+ pmu_state->vals = calloc(evt_arr_sz + 1, sizeof(uint64_t));
+ pmu_state->evts = evt_arr;
+ pmu_state->evt_cnt = evt_arr_sz;
+}
+
+/**
+ * clean_pmu - Frees memory of value array and reset size
+ *
+ * @pmu_state: current states of the pmu counters
+ */
+static inline void clean_pmu(struct trusty_pmu_state* pmu_state) {
+ free(pmu_state->vals);
+ pmu_state->vals = NULL;
+ pmu_state->evt_cnt = 0;
+}
+
+/**
+ * reset_pmu_cnts - Resets all pmu counters to 0
+ *
+ * @pmu_state: current states of the pmu counters
+ */
+static inline void reset_pmu_cnts(struct trusty_pmu_state* pmu_state) {
+ memset(pmu_state->vals, 0, sizeof(uint64_t) * pmu_state->evt_cnt);
+}
diff --git a/lib/pmu/include/lib/pmu/pmu_arch.h b/lib/pmu/include/lib/pmu/pmu_arch.h
new file mode 100644
index 0000000..689f640
--- /dev/null
+++ b/lib/pmu/include/lib/pmu/pmu_arch.h
@@ -0,0 +1,117 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files
+ * (the "Software"), to deal in the Software without restriction,
+ * including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef PMU_ARCH_H
+#define PMU_ARCH_H
+
+#define U(_x) (_x##U)
+
+/*******************************************************************************
+ * Definitions of register offsets and fields in the CNTBaseN Frame of the
+ * system level implementation of the Generic Timer.
+ ******************************************************************************/
+/* Physical Count register. */
+#define CNTPCT_LO U(0x0)
+/* Counter Frequency register. */
+#define CNTBASEN_CNTFRQ U(0x10)
+/* Physical Timer CompareValue register. */
+#define CNTP_CVAL_LO U(0x20)
+/* Physical Timer Control register. */
+#define CNTP_CTL U(0x2c)
+
+/* PMCR_EL0 definitions */
+#define PMCR_EL0_RESET_VAL U(0x0)
+#define PMCR_EL0_N_SHIFT U(11)
+#define PMCR_EL0_N_MASK U(0x1f)
+#define PMCR_EL0_N_BITS (PMCR_EL0_N_MASK << PMCR_EL0_N_SHIFT)
+#define PMCR_EL0_LC_BIT (U(1) << 6)
+#define PMCR_EL0_DP_BIT (U(1) << 5)
+#define PMCR_EL0_X_BIT (U(1) << 4)
+#define PMCR_EL0_D_BIT (U(1) << 3)
+#define PMCR_EL0_C_BIT (U(1) << 2)
+#define PMCR_EL0_P_BIT (U(1) << 1)
+#define PMCR_EL0_E_BIT (U(1) << 0)
+
+/* PMCNTENSET_EL0 definitions */
+#define PMCNTENSET_EL0_C_BIT (U(1) << 31)
+#define PMCNTENSET_EL0_P_BIT(x) (U(1) << x)
+
+/* PMEVTYPER<n>_EL0 definitions */
+#define PMEVTYPER_EL0_P_BIT (U(1) << 31)
+#define PMEVTYPER_EL0_NSK_BIT (U(1) << 29)
+#define PMEVTYPER_EL0_NSH_BIT (U(1) << 27)
+#define PMEVTYPER_EL0_M_BIT (U(1) << 26)
+#define PMEVTYPER_EL0_MT_BIT (U(1) << 25)
+#define PMEVTYPER_EL0_SH_BIT (U(1) << 24)
+#define PMEVTYPER_EL0_EVTCOUNT_BITS U(0x000003FF)
+
+/* PMCCFILTR_EL0 definitions */
+#define PMCCFILTR_EL0_P_BIT (U(1) << 31)
+#define PMCCFILTR_EL0_U_BIT (U(1) << 30)
+#define PMCCFILTR_EL0_NSK_BIT (U(1) << 29)
+#define PMCCFILTR_EL0_NSH_BIT (U(1) << 27)
+#define PMCCFILTR_EL0_M_BIT (U(1) << 26)
+#define PMCCFILTR_EL0_MT_BIT (U(1) << 25)
+#define PMCCFILTR_EL0_SH_BIT (U(1) << 24)
+#define PMCCFILTR_EL0_T_BIT (U(1) << 23)
+#define PMCCFILTR_EL0_RLK_BIT (U(1) << 22)
+#define PMCCFILTR_EL0_RLU_BIT (U(1) << 21)
+#define PMCCFILTR_EL0_RLH_BIT (U(1) << 20)
+
+/* PMU event counter ID definitions */
+#define PMU_EV_SW_INCR U(0x00)
+#define PMU_EV_L1I_CACHE_REFILL U(0x01)
+#define PMU_EV_L1I_TLB_REFILL U(0x02)
+#define PMU_EV_L1D_CACHE_REFILL U(0x03)
+#define PMU_EV_L1D_CACHE U(0x04)
+#define PMU_EV_L1D_TLB_REFILL U(0x05)
+#define PMU_EV_LD_RETIRED U(0x06)
+#define PMU_EV_ST_RETIRED U(0x07)
+#define PMU_EV_INST_RETIRED U(0x08)
+#define PMU_EV_EXC_TAKEN U(0x09)
+#define PMU_EV_EXC_RETURN U(0x0A)
+#define PMU_EV_CID_WRITE_RETIRED U(0x0B)
+#define PMU_EV_PC_WRITE_RETIRED U(0x0C)
+#define PMU_EV_BR_IMMED_RETIRED U(0x0D)
+#define PMU_EV_BR_RETURN_RETIRED U(0x0E)
+#define PMU_EV_UNALIGNED_LDST_RETIRED U(0x0F)
+#define PMU_EV_BR_MIS_PRED U(0x10)
+#define PMU_EV_CPU_CYCLES U(0x11)
+#define PMU_EV_BR_PRED U(0x12)
+#define PMU_EV_MEM_ACCESS U(0x13)
+#define PMU_EV_L1I_CACHE U(0x14)
+#define PMU_EV_L1D_CACHE_WB U(0x15)
+#define PMU_EV_L2D_CACHE U(0x16)
+#define PMU_EV_L2D_CACHE_REFILL U(0x17)
+#define PMU_EV_L2D_CACHE_WB U(0x18)
+#define PMU_EV_BUS_ACCESS U(0x19)
+#define PMU_EV_MEMORY_ERROR U(0x1A)
+#define PMU_EV_BUS_CYCLES U(0x1D)
+#define PMU_EV_CHAIN U(0x1E)
+#define PMU_EV_BUS_ACCESS_LD U(0x60)
+#define PMU_EV_BUS_ACCESS_ST U(0x61)
+#define PMU_EV_BR_INDIRECT_SPEC U(0x7A)
+#define PMU_EV_EXC_IRQ U(0x86)
+#define PMU_EV_EXC_FIQ U(0x87)
+
+#endif /* PMU_ARCH_H */
diff --git a/lib/pmu/rules.mk b/lib/pmu/rules.mk
new file mode 100644
index 0000000..7050fd5
--- /dev/null
+++ b/lib/pmu/rules.mk
@@ -0,0 +1,29 @@
+# Copyright (C) 2023 The Android Open Source Project
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files
+# (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge,
+# publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+
+LOCAL_DIR := $(GET_LOCAL_DIR)
+
+MODULE := $(LOCAL_DIR)
+
+GLOBAL_INCLUDES += $(LOCAL_DIR)/include/
+
+include make/module.mk
diff --git a/lib/trusty/trusty_virtio.c b/lib/trusty/trusty_virtio.c
index 615b9ff..7f8616d 100644
--- a/lib/trusty/trusty_virtio.c
+++ b/lib/trusty/trusty_virtio.c
@@ -504,7 +504,7 @@
goto err_bad_state;
}
- if ((size_t)descr_sz != vb->descr_size) {
+ if ((size_t)descr_sz != vb->descr_size || descr_sz == 0) {
LTRACEF("unexpected descriptor size (%zd vs. %zd)\n", (size_t)descr_sz,
vb->descr_size);
ret = ERR_INVALID_ARGS;
diff --git a/lib/unittest/rules.mk b/lib/unittest/rules.mk
index e6baedd..bf73fb7 100644
--- a/lib/unittest/rules.mk
+++ b/lib/unittest/rules.mk
@@ -24,9 +24,11 @@
MODULE := $(LOCAL_DIR)
-MODULE_SRCS := \
- $(LOCAL_DIR)/unittest.c
+MODULE_SRCS := $(LOCAL_DIR)/unittest.c
-GLOBAL_INCLUDES += $(LOCAL_DIR)/include/
+GLOBAL_INCLUDES += $(LOCAL_DIR)/include/ \
+ $(LOCAL_DIR)/../pmu/include/
+
+MODULE_DEPS += trusty/kernel/lib/pmu
include make/module.mk
diff --git a/platform/generic-x86_64/rules.mk b/platform/generic-x86_64/rules.mk
index c32ef81..ca9811a 100644
--- a/platform/generic-x86_64/rules.mk
+++ b/platform/generic-x86_64/rules.mk
@@ -30,6 +30,5 @@
$(LOCAL_DIR)/rust \
dev/interrupt/x86_lapic \
dev/timer/x86_generic \
- dev/virtio/vsock-rust \
include make/module.mk
diff --git a/platform/generic-x86_64/rust/rules.mk b/platform/generic-x86_64/rust/rules.mk
index 6a71969..40c05fd 100644
--- a/platform/generic-x86_64/rust/rules.mk
+++ b/platform/generic-x86_64/rust/rules.mk
@@ -10,5 +10,6 @@
MODULE_LIBRARY_DEPS += \
external/rust/crates/acpi \
external/rust/crates/log \
+ dev/virtio/vsock-rust \
include make/library.mk
diff --git a/platform/generic-x86_64/rust/src/lib.rs b/platform/generic-x86_64/rust/src/lib.rs
index 67b3ca0..2c78b62 100644
--- a/platform/generic-x86_64/rust/src/lib.rs
+++ b/platform/generic-x86_64/rust/src/lib.rs
@@ -86,7 +86,14 @@
}
}
-extern "C" fn platform_acpi_init_func(_level: c_uint) {
+/// Search the ACPI tables for the physical address which is the base of the MMIO
+/// region and pass it to `pci_init_mmio` to initialize the virtio-vsock bridge.
+///
+/// # Safety
+///
+/// A caller must ensure that this function is called exactly once during kernel
+/// initialization on a system which has a BIOS (as opposed to UEFI systems).
+unsafe extern "C" fn platform_acpi_init_func(_level: c_uint) {
// SAFETY: search_for_rsdp_bios searches for a RSDP on BIOS systems.
// It is not safe to call on a UEFI system. crosvm currently emulates
// a BIOS system.
@@ -109,10 +116,10 @@
let entry_size = (1 + entry.bus_number_end as usize - entry.bus_number_start as usize) << 20;
- log::error!(
- "TODO: call init function for pci bus at {:#x}, size {entry_size:#x}",
- entry.base_address as usize
- );
+ // SAFETY: the first argument must be a valid physical address pointing to the base of the MMIO region.
+ // The second argument must be the size of the MMIO region and be valid under the PCI express version
+ // implemented by the system. `pci_init_mmio` must not have been called previously.
+ unsafe { vsock::pci_init_mmio(entry.base_address as usize, entry_size, 1usize << 15) };
}
LK_INIT_HOOK!(platform_acpi_init, platform_acpi_init_func, lk_init_level::LK_INIT_LEVEL_THREADING);
