| From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 |
| From: JianMin Liu <jian-min.liu@mediatek.com> |
| Date: Tue, 21 Jul 2020 13:36:22 +0800 |
| Subject: NOUPSTREAM: ANDROID: kernel: Core kernel related vendor hooks |
| |
| - Add the hook to apply vendor's performance tune for owner |
| of rwsem. |
| |
| - Add the hook for the waiter list of rwsem to allow |
| vendor perform waiting queue enhancement |
| |
| - ANDROID_VENDOR_DATA added to rw_semaphore |
| |
| [CPNOTE: 27/05/21] Lee: Vendor related code - maintain forever |
| |
| Squash: |
| NOUPSTREAM: ANDROID: workqueue: add vendor hook for wq lockup information |
| NOUPSTREAM: ANDROID: printk: add vendor hook for console flush |
| NOUPSTREAM: ANDROID: vendor_hooks: Add hook for jiffies updates |
| NOUPSTREAM: ANDROID: sched_clock: Add vendor hook to print epoch values |
| NOUPSTREAM: ANDROID: sched/cpuset: Add vendor hook to change tasks affinity |
| NOUPSTREAM: ANDROID: tracing: Add restricted vendor hooks for preemptirq events |
| NOUPSTREAM: ANDROID: ftrace: vendor hook for ftrace dump on oops |
| NOUPSTREAM: ANDROID: cgroup: Add vendor hook to the cgroup |
| NOUPSTREAM: ANDROID: Add vendor hooks when syscall prctl finished |
| NOUPSTREAM: ANDROID: timer: Add vendor hook for timer calc index |
| ANDROID: cgroup: Add android_rvh_cgroup_force_kthread_migration |
| ANDROID: kernel: Add restricted vendor hook in creds |
| |
| Bug: 161400830 |
| Bug: 169374262 |
| Bug: 165340180 |
| Bug: 148928265 |
| Bug: 172945021 |
| Bug: 174125747 |
| Bug: 174541725 |
| Bug: 177387994 |
| Bug: 181917687 |
| Bug: 181819699 |
| Bug: 178758017 |
| Bug: 184594949 |
| Bug: 181639260 |
| |
| Signed-off-by: JianMin Liu <jian-min.liu@mediatek.com> |
| Change-Id: I007a5e26f3db2adaeaf4e5ccea414ce7abfa83b8 |
| |
| ANDROID: module: Add vendor hook |
| |
| Add vendor hook for module init, so we can get memory type and |
| use it to do memory type check for architecture |
| dependent page table setting. |
| |
| Bug: 181639260 |
| |
| Signed-off-by: Kuan-Ying Lee <Kuan-Ying.Lee@mediatek.com> |
| Change-Id: I95b70d7a57994f2548fddfb2290d4c9136f58785 |
| |
| ANDROID: syscall_check: add vendor hook for bpf syscall |
| |
| Through this vendor hook, we can get the timing to check |
| current running task for the validation of its credential |
| and bpf operations. |
| |
| Bug: 191291287 |
| |
| Signed-off-by: Kuan-Ying Lee <Kuan-Ying.Lee@mediatek.com> |
| Change-Id: Ie4ed8df7ad66df2486fc7e52a26d9191fc0c176e |
| (cherry picked from commit 38abaebab795d87a4e5bbf2ce00e7e37a3db3847) |
| --- |
| drivers/android/vendor_hooks.c | 48 ++++++++++++++++++++++++--- |
| include/trace/hooks/cgroup.h | 26 +++++++++++++++ |
| include/trace/hooks/creds.h | 34 ++++++++++++++++++++ |
| include/trace/hooks/dtask.h | 14 -------- |
| include/trace/hooks/epoch.h | 23 +++++++++++++ |
| include/trace/hooks/ftrace_dump.h | 50 +++++++++++++++++++++++++++++ |
| include/trace/hooks/memory.h | 24 ++++++++++++++ |
| include/trace/hooks/module.h | 25 +++++++++++++++ |
| include/trace/hooks/preemptirq.h | 32 ++++++++++++++++++ |
| include/trace/hooks/printk.h | 19 +++++++++++ |
| include/trace/hooks/sched.h | 9 ++++++ |
| include/trace/hooks/sys.h | 17 ++++++++++ |
| include/trace/hooks/syscall_check.h | 5 +++ |
| include/trace/hooks/timer.h | 19 +++++++++++ |
| include/trace/hooks/wqlockup.h | 20 ++++++++++++ |
| kernel/bpf/syscall.c | 4 +++ |
| kernel/cgroup/cgroup-internal.h | 3 +- |
| kernel/cgroup/cgroup-v1.c | 4 ++- |
| kernel/cgroup/cgroup.c | 15 +++++++-- |
| kernel/cgroup/cpuset.c | 18 +++++++++-- |
| kernel/cred.c | 6 ++++ |
| kernel/locking/rwsem.c | 7 ---- |
| kernel/module.c | 26 +++++++++++++++ |
| kernel/printk/printk.c | 8 +++++ |
| kernel/sys.c | 3 ++ |
| kernel/time/sched_clock.c | 3 ++ |
| kernel/time/tick-common.c | 2 ++ |
| kernel/time/tick-sched.c | 5 ++- |
| kernel/time/timer.c | 3 ++ |
| kernel/trace/trace.c | 37 +++++++++++++++++++-- |
| kernel/trace/trace_preemptirq.c | 40 ++++++++++++++++++----- |
| kernel/workqueue.c | 5 +++ |
| 32 files changed, 511 insertions(+), 43 deletions(-) |
| create mode 100644 include/trace/hooks/cgroup.h |
| create mode 100644 include/trace/hooks/creds.h |
| create mode 100644 include/trace/hooks/epoch.h |
| create mode 100644 include/trace/hooks/ftrace_dump.h |
| create mode 100644 include/trace/hooks/memory.h |
| create mode 100644 include/trace/hooks/module.h |
| create mode 100644 include/trace/hooks/preemptirq.h |
| create mode 100644 include/trace/hooks/printk.h |
| create mode 100644 include/trace/hooks/sys.h |
| create mode 100644 include/trace/hooks/timer.h |
| create mode 100644 include/trace/hooks/wqlockup.h |
| |
| diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c |
| --- a/drivers/android/vendor_hooks.c |
| +++ b/drivers/android/vendor_hooks.c |
| @@ -12,10 +12,22 @@ |
| #include <trace/hooks/fpsimd.h> |
| #include <trace/hooks/dtask.h> |
| #include <trace/hooks/mpam.h> |
| +#include <trace/hooks/wqlockup.h> |
| #include <trace/hooks/debug.h> |
| +#include <trace/hooks/printk.h> |
| +#include <trace/hooks/epoch.h> |
| #include <trace/hooks/mm.h> |
| +#include <trace/hooks/preemptirq.h> |
| +#include <trace/hooks/ftrace_dump.h> |
| +#include <trace/hooks/cgroup.h> |
| +#include <trace/hooks/sys.h> |
| #include <trace/hooks/iommu.h> |
| +#include <trace/hooks/net.h> |
| +#include <trace/hooks/timer.h> |
| #include <trace/hooks/vmscan.h> |
| +#include <trace/hooks/creds.h> |
| +#include <trace/hooks/memory.h> |
| +#include <trace/hooks/module.h> |
| #include <trace/hooks/syscall_check.h> |
| |
| /* |
| @@ -36,14 +48,16 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_finish_prio_fork); |
| EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_user_nice); |
| EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_setscheduler); |
| EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_is_fpsimd_save); |
| -EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_read_wait_start); |
| -EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_read_wait_finish); |
| -EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_write_wait_start); |
| -EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_write_wait_finish); |
| EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_sched_show_task); |
| EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mpam_set); |
| +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_find_busiest_group); |
| +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_wq_lockup_pool); |
| EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ipi_stop); |
| EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_dump_throttled_rt_tasks); |
| +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_printk_hotplug); |
| +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_jiffies_update); |
| +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_show_suspend_epoch_val); |
| +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_show_resume_epoch_val); |
| EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_sched_newidle_balance); |
| EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_sched_nohz_balancer_kick); |
| EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_find_busiest_queue); |
| @@ -53,9 +67,14 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_set_sugov_sched_attr); |
| EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_iowait); |
| EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_sugov_update); |
| EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_sched_setaffinity); |
| +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_update_cpus_allowed); |
| EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_skip_swapcache_flags); |
| EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_gfp_zone_flags); |
| EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_readahead_gfp_mask); |
| +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_preempt_disable); |
| +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_preempt_enable); |
| +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_irqs_disable); |
| +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_irqs_enable); |
| EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_task_cpu); |
| EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_try_to_wake_up); |
| EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_try_to_wake_up_success); |
| @@ -81,7 +100,19 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_schedule_bug); |
| EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_sched_exec); |
| EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_map_util_freq); |
| EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_em_cpu_energy); |
| +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ftrace_oops_enter); |
| +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ftrace_oops_exit); |
| +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ftrace_size_check); |
| +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ftrace_format_check); |
| +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ftrace_dump_buffer); |
| +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cgroup_set_task); |
| +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_syscall_prctl_finished); |
| +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cgroup_attach); |
| EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_iommu_setup_dma_ops); |
| +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ptype_head); |
| +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_kfree_skb); |
| +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_timer_calc_index); |
| +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_cgroup_force_kthread_migration); |
| EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_check_preempt_tick); |
| EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_check_preempt_wakeup_ignore); |
| EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_replace_next_task_fair); |
| @@ -98,4 +129,13 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_dequeue_entity); |
| EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_entity_tick); |
| EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_enqueue_task_fair); |
| EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_dequeue_task_fair); |
| +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_commit_creds); |
| +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_exit_creds); |
| +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_override_creds); |
| +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_revert_creds); |
| +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_set_memory_nx); |
| +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_set_memory_rw); |
| +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_set_module_permit_before_init); |
| +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_set_module_permit_after_init); |
| EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_check_mmap_file); |
| +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_check_bpf_syscall); |
| diff --git a/include/trace/hooks/cgroup.h b/include/trace/hooks/cgroup.h |
| new file mode 100644 |
| --- /dev/null |
| +++ b/include/trace/hooks/cgroup.h |
| @@ -0,0 +1,26 @@ |
| +/* SPDX-License-Identifier: GPL-2.0 */ |
| +#undef TRACE_SYSTEM |
| +#define TRACE_SYSTEM cgroup |
| +#undef TRACE_INCLUDE_PATH |
| +#define TRACE_INCLUDE_PATH trace/hooks |
| +#if !defined(_TRACE_HOOK_CGROUP_H) || defined(TRACE_HEADER_MULTI_READ) |
| +#define _TRACE_HOOK_CGROUP_H |
| +#include <linux/tracepoint.h> |
| +#include <trace/hooks/vendor_hooks.h> |
| + |
| +struct task_struct; |
| +DECLARE_HOOK(android_vh_cgroup_set_task, |
| + TP_PROTO(int ret, struct task_struct *task), |
| + TP_ARGS(ret, task)); |
| + |
| +struct cgroup_subsys; |
| +struct cgroup_taskset; |
| +DECLARE_HOOK(android_vh_cgroup_attach, |
| + TP_PROTO(struct cgroup_subsys *ss, struct cgroup_taskset *tset), |
| + TP_ARGS(ss, tset)) |
| +DECLARE_RESTRICTED_HOOK(android_rvh_cgroup_force_kthread_migration, |
| + TP_PROTO(struct task_struct *tsk, struct cgroup *dst_cgrp, bool *force_migration), |
| + TP_ARGS(tsk, dst_cgrp, force_migration), 1); |
| +#endif |
| + |
| +#include <trace/define_trace.h> |
| diff --git a/include/trace/hooks/creds.h b/include/trace/hooks/creds.h |
| new file mode 100644 |
| --- /dev/null |
| +++ b/include/trace/hooks/creds.h |
| @@ -0,0 +1,34 @@ |
| +/* SPDX-License-Identifier: GPL-2.0 */ |
| +#undef TRACE_SYSTEM |
| +#define TRACE_SYSTEM creds |
| + |
| +#define TRACE_INCLUDE_PATH trace/hooks |
| +#if !defined(_TRACE_HOOK_CREDS_H) || defined(TRACE_HEADER_MULTI_READ) |
| +#define _TRACE_HOOK_CREDS_H |
| +#include <linux/tracepoint.h> |
| +#include <trace/hooks/vendor_hooks.h> |
| +/* |
| + * Following tracepoints are not exported in tracefs and provide a |
| + * mechanism for vendor modules to hook and extend functionality |
| + */ |
| +struct cred; |
| +struct task_struct; |
| +DECLARE_RESTRICTED_HOOK(android_rvh_commit_creds, |
| + TP_PROTO(const struct task_struct *task, const struct cred *new), |
| + TP_ARGS(task, new), 1); |
| + |
| +DECLARE_RESTRICTED_HOOK(android_rvh_exit_creds, |
| + TP_PROTO(const struct task_struct *task, const struct cred *cred), |
| + TP_ARGS(task, cred), 1); |
| + |
| +DECLARE_RESTRICTED_HOOK(android_rvh_override_creds, |
| + TP_PROTO(const struct task_struct *task, const struct cred *new), |
| + TP_ARGS(task, new), 1); |
| + |
| +DECLARE_RESTRICTED_HOOK(android_rvh_revert_creds, |
| + TP_PROTO(const struct task_struct *task, const struct cred *old), |
| + TP_ARGS(task, old), 1); |
| + |
| +#endif /* _TRACE_HOOK_CREDS_H */ |
| +/* This part must be outside protection */ |
| +#include <trace/define_trace.h> |
| diff --git a/include/trace/hooks/dtask.h b/include/trace/hooks/dtask.h |
| --- a/include/trace/hooks/dtask.h |
| +++ b/include/trace/hooks/dtask.h |
| @@ -11,20 +11,6 @@ |
| * Following tracepoints are not exported in tracefs and provide a |
| * mechanism for vendor modules to hook and extend functionality |
| */ |
| -struct rw_semaphore; |
| -DECLARE_HOOK(android_vh_rwsem_read_wait_start, |
| - TP_PROTO(struct rw_semaphore *sem), |
| - TP_ARGS(sem)); |
| -DECLARE_HOOK(android_vh_rwsem_read_wait_finish, |
| - TP_PROTO(struct rw_semaphore *sem), |
| - TP_ARGS(sem)); |
| -DECLARE_HOOK(android_vh_rwsem_write_wait_start, |
| - TP_PROTO(struct rw_semaphore *sem), |
| - TP_ARGS(sem)); |
| -DECLARE_HOOK(android_vh_rwsem_write_wait_finish, |
| - TP_PROTO(struct rw_semaphore *sem), |
| - TP_ARGS(sem)); |
| - |
| struct task_struct; |
| DECLARE_HOOK(android_vh_sched_show_task, |
| TP_PROTO(struct task_struct *task), |
| diff --git a/include/trace/hooks/epoch.h b/include/trace/hooks/epoch.h |
| new file mode 100644 |
| --- /dev/null |
| +++ b/include/trace/hooks/epoch.h |
| @@ -0,0 +1,23 @@ |
| +/* SPDX-License-Identifier: GPL-2.0 */ |
| +#undef TRACE_SYSTEM |
| +#define TRACE_SYSTEM epoch |
| + |
| +#define TRACE_INCLUDE_PATH trace/hooks |
| + |
| +#if !defined(_TRACE_HOOK_EPOCH_H) || defined(TRACE_HEADER_MULTI_READ) |
| +#define _TRACE_HOOK_EPOCH_H |
| + |
| +#include <linux/tracepoint.h> |
| +#include <trace/hooks/vendor_hooks.h> |
| + |
| +DECLARE_HOOK(android_vh_show_suspend_epoch_val, |
| + TP_PROTO(u64 suspend_ns, u64 suspend_cycles), |
| + TP_ARGS(suspend_ns, suspend_cycles)); |
| + |
| +DECLARE_HOOK(android_vh_show_resume_epoch_val, |
| + TP_PROTO(u64 resume_cycles), |
| + TP_ARGS(resume_cycles)); |
| + |
| +#endif /* _TRACE_HOOK_EPOCH_H */ |
| +/* This part must be outside protection */ |
| +#include <trace/define_trace.h> |
| diff --git a/include/trace/hooks/ftrace_dump.h b/include/trace/hooks/ftrace_dump.h |
| new file mode 100644 |
| --- /dev/null |
| +++ b/include/trace/hooks/ftrace_dump.h |
| @@ -0,0 +1,50 @@ |
| +/* SPDX-License-Identifier: GPL-2.0 */ |
| +#undef TRACE_SYSTEM |
| +#define TRACE_SYSTEM ftrace_dump |
| + |
| +#define TRACE_INCLUDE_PATH trace/hooks |
| + |
| +#if !defined(_TRACE_HOOK_FTRACE_DUMP_H) || defined(TRACE_HEADER_MULTI_READ) |
| +#define _TRACE_HOOK_FTRACE_DUMP_H |
| + |
| +#include <linux/trace_seq.h> |
| +#include <linux/trace_events.h> |
| + |
| +#include <linux/tracepoint.h> |
| +#include <trace/hooks/vendor_hooks.h> |
| + |
| +#if defined(CONFIG_TRACEPOINTS) && defined(CONFIG_ANDROID_VENDOR_HOOKS) |
| + |
| +DECLARE_HOOK(android_vh_ftrace_oops_enter, |
| + TP_PROTO(bool *ftrace_check), |
| + TP_ARGS(ftrace_check)); |
| + |
| +DECLARE_HOOK(android_vh_ftrace_oops_exit, |
| + TP_PROTO(bool *ftrace_check), |
| + TP_ARGS(ftrace_check)); |
| + |
| +DECLARE_HOOK(android_vh_ftrace_size_check, |
| + TP_PROTO(unsigned long size, bool *ftrace_check), |
| + TP_ARGS(size, ftrace_check)); |
| + |
| +DECLARE_HOOK(android_vh_ftrace_format_check, |
| + TP_PROTO(bool *ftrace_check), |
| + TP_ARGS(ftrace_check)); |
| + |
| +DECLARE_HOOK(android_vh_ftrace_dump_buffer, |
| + TP_PROTO(struct trace_seq *trace_buf, bool *dump_printk), |
| + TP_ARGS(trace_buf, dump_printk)); |
| + |
| +#else |
| + |
| +#define trace_android_vh_ftrace_oops_enter(ftrace_check) |
| +#define trace_android_vh_ftrace_oops_exit(ftrace_check) |
| +#define trace_android_vh_ftrace_size_check(size, ftrace_check) |
| +#define trace_android_vh_ftrace_format_check(ftrace_check) |
| +#define trace_android_vh_ftrace_dump_buffer(trace_buf, dump_printk) |
| + |
| +#endif |
| + |
| +#endif /* _TRACE_HOOK_FTRACE_DUMP_H */ |
| +/* This part must be outside protection */ |
| +#include <trace/define_trace.h> |
| diff --git a/include/trace/hooks/memory.h b/include/trace/hooks/memory.h |
| new file mode 100644 |
| --- /dev/null |
| +++ b/include/trace/hooks/memory.h |
| @@ -0,0 +1,24 @@ |
| +/* SPDX-License-Identifier: GPL-2.0 */ |
| +#undef TRACE_SYSTEM |
| +#define TRACE_SYSTEM memory |
| + |
| +#define TRACE_INCLUDE_PATH trace/hooks |
| +#if !defined(_TRACE_HOOK_MEMORY_H) || defined(TRACE_HEADER_MULTI_READ) |
| +#define _TRACE_HOOK_MEMORY_H |
| +#include <linux/tracepoint.h> |
| +#include <trace/hooks/vendor_hooks.h> |
| +/* |
| + * Following tracepoints are not exported in tracefs and provide a |
| + * mechanism for vendor modules to hook and extend functionality |
| + */ |
| +DECLARE_HOOK(android_vh_set_memory_nx, |
| + TP_PROTO(unsigned long addr, int nr_pages), |
| + TP_ARGS(addr, nr_pages)); |
| + |
| +DECLARE_HOOK(android_vh_set_memory_rw, |
| + TP_PROTO(unsigned long addr, int nr_pages), |
| + TP_ARGS(addr, nr_pages)); |
| + |
| +#endif /* _TRACE_HOOK_MEMORY_H */ |
| +/* This part must be outside protection */ |
| +#include <trace/define_trace.h> |
| diff --git a/include/trace/hooks/module.h b/include/trace/hooks/module.h |
| new file mode 100644 |
| --- /dev/null |
| +++ b/include/trace/hooks/module.h |
| @@ -0,0 +1,25 @@ |
| +/* SPDX-License-Identifier: GPL-2.0 */ |
| +#undef TRACE_SYSTEM |
| +#define TRACE_SYSTEM module |
| + |
| +#define TRACE_INCLUDE_PATH trace/hooks |
| +#if !defined(_TRACE_HOOK_MODULE_H) || defined(TRACE_HEADER_MULTI_READ) |
| +#define _TRACE_HOOK_MODULE_H |
| +#include <linux/tracepoint.h> |
| +#include <trace/hooks/vendor_hooks.h> |
| +/* |
| + * Following tracepoints are not exported in tracefs and provide a |
| + * mechanism for vendor modules to hook and extend functionality |
| + */ |
| +struct module; |
| +DECLARE_HOOK(android_vh_set_module_permit_before_init, |
| + TP_PROTO(const struct module *mod), |
| + TP_ARGS(mod)); |
| + |
| +DECLARE_HOOK(android_vh_set_module_permit_after_init, |
| + TP_PROTO(const struct module *mod), |
| + TP_ARGS(mod)); |
| + |
| +#endif /* _TRACE_HOOK_MODULE_H */ |
| +/* This part must be outside protection */ |
| +#include <trace/define_trace.h> |
| diff --git a/include/trace/hooks/preemptirq.h b/include/trace/hooks/preemptirq.h |
| new file mode 100644 |
| --- /dev/null |
| +++ b/include/trace/hooks/preemptirq.h |
| @@ -0,0 +1,32 @@ |
| +/* SPDX-License-Identifier: GPL-2.0 */ |
| +#undef TRACE_SYSTEM |
| +#define TRACE_SYSTEM preemptirq |
| + |
| +#undef TRACE_INCLUDE_PATH |
| +#define TRACE_INCLUDE_PATH trace/hooks |
| + |
| +#if !defined(_TRACE_HOOK_PREEMPTIRQ_H) || defined(TRACE_HEADER_MULTI_READ) |
| +#define _TRACE_HOOK_PREEMPTIRQ_H |
| + |
| +#include <linux/tracepoint.h> |
| +#include <trace/hooks/vendor_hooks.h> |
| + |
| +DECLARE_RESTRICTED_HOOK(android_rvh_preempt_disable, |
| + TP_PROTO(unsigned long ip, unsigned long parent_ip), |
| + TP_ARGS(ip, parent_ip), 1); |
| + |
| +DECLARE_RESTRICTED_HOOK(android_rvh_preempt_enable, |
| + TP_PROTO(unsigned long ip, unsigned long parent_ip), |
| + TP_ARGS(ip, parent_ip), 1); |
| + |
| +DECLARE_RESTRICTED_HOOK(android_rvh_irqs_disable, |
| + TP_PROTO(unsigned long ip, unsigned long parent_ip), |
| + TP_ARGS(ip, parent_ip), 1); |
| + |
| +DECLARE_RESTRICTED_HOOK(android_rvh_irqs_enable, |
| + TP_PROTO(unsigned long ip, unsigned long parent_ip), |
| + TP_ARGS(ip, parent_ip), 1); |
| + |
| +#endif /* _TRACE_HOOK_PREEMPTIRQ_H */ |
| +/* This part must be outside protection */ |
| +#include <trace/define_trace.h> |
| diff --git a/include/trace/hooks/printk.h b/include/trace/hooks/printk.h |
| new file mode 100644 |
| --- /dev/null |
| +++ b/include/trace/hooks/printk.h |
| @@ -0,0 +1,19 @@ |
| +/* SPDX-License-Identifier: GPL-2.0 */ |
| +#undef TRACE_SYSTEM |
| +#define TRACE_SYSTEM printk |
| + |
| +#define TRACE_INCLUDE_PATH trace/hooks |
| + |
| +#if !defined(_TRACE_HOOK_PRINTK_H) || defined(TRACE_HEADER_MULTI_READ) |
| +#define _TRACE_HOOK_PRINTK_H |
| + |
| +#include <linux/tracepoint.h> |
| +#include <trace/hooks/vendor_hooks.h> |
| + |
| +DECLARE_HOOK(android_vh_printk_hotplug, |
| + TP_PROTO(int *flag), |
| + TP_ARGS(flag)); |
| + |
| +#endif /* _TRACE_HOOK_PRINTK_H */ |
| +/* This part must be outside protection */ |
| +#include <trace/define_trace.h> |
| diff --git a/include/trace/hooks/sched.h b/include/trace/hooks/sched.h |
| --- a/include/trace/hooks/sched.h |
| +++ b/include/trace/hooks/sched.h |
| @@ -75,6 +75,10 @@ DECLARE_HOOK(android_vh_dump_throttled_rt_tasks, |
| s64 rt_period_timer_expires), |
| TP_ARGS(cpu, clock, rt_period, rt_runtime, rt_period_timer_expires)); |
| |
| +DECLARE_HOOK(android_vh_jiffies_update, |
| + TP_PROTO(void *unused), |
| + TP_ARGS(unused)); |
| + |
| struct rq_flags; |
| DECLARE_RESTRICTED_HOOK(android_rvh_sched_newidle_balance, |
| TP_PROTO(struct rq *this_rq, struct rq_flags *rf, |
| @@ -115,6 +119,11 @@ DECLARE_RESTRICTED_HOOK(android_rvh_sched_setaffinity, |
| TP_PROTO(struct task_struct *p, const struct cpumask *in_mask, int *retval), |
| TP_ARGS(p, in_mask, retval), 1); |
| |
| +DECLARE_RESTRICTED_HOOK(android_rvh_update_cpus_allowed, |
| + TP_PROTO(struct task_struct *p, cpumask_var_t cpus_requested, |
| + const struct cpumask *new_mask, int *ret), |
| + TP_ARGS(p, cpus_requested, new_mask, ret), 1); |
| + |
| DECLARE_RESTRICTED_HOOK(android_rvh_set_task_cpu, |
| TP_PROTO(struct task_struct *p, unsigned int new_cpu), |
| TP_ARGS(p, new_cpu), 1); |
| diff --git a/include/trace/hooks/sys.h b/include/trace/hooks/sys.h |
| new file mode 100644 |
| --- /dev/null |
| +++ b/include/trace/hooks/sys.h |
| @@ -0,0 +1,17 @@ |
| +/* SPDX-License-Identifier: GPL-2.0 */ |
| +#undef TRACE_SYSTEM |
| +#define TRACE_SYSTEM sys |
| +#undef TRACE_INCLUDE_PATH |
| +#define TRACE_INCLUDE_PATH trace/hooks |
| +#if !defined(_TRACE_HOOK_SYS_H) || defined(TRACE_HEADER_MULTI_READ) |
| +#define _TRACE_HOOK_SYS_H |
| +#include <linux/tracepoint.h> |
| +#include <trace/hooks/vendor_hooks.h> |
| + |
| +struct task_struct; |
| +DECLARE_HOOK(android_vh_syscall_prctl_finished, |
| + TP_PROTO(int option, struct task_struct *task), |
| + TP_ARGS(option, task)); |
| +#endif |
| + |
| +#include <trace/define_trace.h> |
| diff --git a/include/trace/hooks/syscall_check.h b/include/trace/hooks/syscall_check.h |
| --- a/include/trace/hooks/syscall_check.h |
| +++ b/include/trace/hooks/syscall_check.h |
| @@ -12,11 +12,16 @@ |
| * mechanism for vendor modules to hook and extend functionality |
| */ |
| struct file; |
| +union bpf_attr; |
| DECLARE_HOOK(android_vh_check_mmap_file, |
| TP_PROTO(const struct file *file, unsigned long prot, |
| unsigned long flag, unsigned long ret), |
| TP_ARGS(file, prot, flag, ret)); |
| |
| +DECLARE_HOOK(android_vh_check_bpf_syscall, |
| + TP_PROTO(int cmd, const union bpf_attr *attr, unsigned int size), |
| + TP_ARGS(cmd, attr, size)); |
| + |
| #endif /* _TRACE_HOOK_SYSCALL_CHECK_H */ |
| /* This part must be outside protection */ |
| #include <trace/define_trace.h> |
| diff --git a/include/trace/hooks/timer.h b/include/trace/hooks/timer.h |
| new file mode 100644 |
| --- /dev/null |
| +++ b/include/trace/hooks/timer.h |
| @@ -0,0 +1,19 @@ |
| +/* SPDX-License-Identifier: GPL-2.0 */ |
| +#undef TRACE_SYSTEM |
| +#define TRACE_SYSTEM timer |
| + |
| +#define TRACE_INCLUDE_PATH trace/hooks |
| + |
| +#if !defined(_TRACE_HOOK_TIMER_H) || defined(TRACE_HEADER_MULTI_READ) |
| +#define _TRACE_HOOK_TIMER_H |
| + |
| +#include <linux/tracepoint.h> |
| +#include <trace/hooks/vendor_hooks.h> |
| + |
| +DECLARE_HOOK(android_vh_timer_calc_index, |
| + TP_PROTO(unsigned int lvl, unsigned long *expires), |
| + TP_ARGS(lvl, expires)); |
| + |
| +#endif /* _TRACE_HOOK_TIMER_H */ |
| +/* This part must be outside protection */ |
| +#include <trace/define_trace.h> |
| diff --git a/include/trace/hooks/wqlockup.h b/include/trace/hooks/wqlockup.h |
| new file mode 100644 |
| --- /dev/null |
| +++ b/include/trace/hooks/wqlockup.h |
| @@ -0,0 +1,20 @@ |
| +/* SPDX-License-Identifier: GPL-2.0 */ |
| +#undef TRACE_SYSTEM |
| +#define TRACE_SYSTEM wqlockup |
| +#define TRACE_INCLUDE_PATH trace/hooks |
| + |
| +#if !defined(_TRACE_HOOK_WQLOCKUP_H) || defined(TRACE_HEADER_MULTI_READ) |
| +#define _TRACE_HOOK_WQLOCKUP_H |
| +#include <linux/tracepoint.h> |
| +#include <trace/hooks/vendor_hooks.h> |
| +/* |
| + * Following tracepoints are not exported in tracefs and provide a |
| + * mechanism for vendor modules to hook and extend functionality |
| + */ |
| +DECLARE_HOOK(android_vh_wq_lockup_pool, |
| + TP_PROTO(int cpu, unsigned long pool_ts), |
| + TP_ARGS(cpu, pool_ts)); |
| + |
| +#endif /* _TRACE_HOOK_WQLOCKUP_H */ |
| +/* This part must be outside protection */ |
| +#include <trace/define_trace.h> |
| diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c |
| --- a/kernel/bpf/syscall.c |
| +++ b/kernel/bpf/syscall.c |
| @@ -32,6 +32,8 @@ |
| #include <linux/rcupdate_trace.h> |
| #include <linux/memcontrol.h> |
| |
| +#include <trace/hooks/syscall_check.h> |
| + |
| #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \ |
| (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \ |
| (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) |
| @@ -4606,6 +4608,8 @@ static int __sys_bpf(int cmd, bpfptr_t uattr, unsigned int size) |
| if (copy_from_bpfptr(&attr, uattr, size) != 0) |
| return -EFAULT; |
| |
| + trace_android_vh_check_bpf_syscall(cmd, &attr, size); |
| + |
| err = security_bpf(cmd, &attr, size); |
| if (err < 0) |
| return err; |
| diff --git a/kernel/cgroup/cgroup-internal.h b/kernel/cgroup/cgroup-internal.h |
| --- a/kernel/cgroup/cgroup-internal.h |
| +++ b/kernel/cgroup/cgroup-internal.h |
| @@ -232,7 +232,8 @@ int cgroup_migrate(struct task_struct *leader, bool threadgroup, |
| int cgroup_attach_task(struct cgroup *dst_cgrp, struct task_struct *leader, |
| bool threadgroup); |
| struct task_struct *cgroup_procs_write_start(char *buf, bool threadgroup, |
| - bool *locked) |
| + bool *locked, |
| + struct cgroup *dst_cgrp); |
| __acquires(&cgroup_threadgroup_rwsem); |
| void cgroup_procs_write_finish(struct task_struct *task, bool locked) |
| __releases(&cgroup_threadgroup_rwsem); |
| diff --git a/kernel/cgroup/cgroup-v1.c b/kernel/cgroup/cgroup-v1.c |
| --- a/kernel/cgroup/cgroup-v1.c |
| +++ b/kernel/cgroup/cgroup-v1.c |
| @@ -17,6 +17,7 @@ |
| #include <linux/fs_parser.h> |
| |
| #include <trace/events/cgroup.h> |
| +#include <trace/hooks/cgroup.h> |
| |
| /* |
| * pidlists linger the following amount before being destroyed. The goal |
| @@ -498,7 +499,7 @@ static ssize_t __cgroup1_procs_write(struct kernfs_open_file *of, |
| if (!cgrp) |
| return -ENODEV; |
| |
| - task = cgroup_procs_write_start(buf, threadgroup, &locked); |
| + task = cgroup_procs_write_start(buf, threadgroup, &locked, cgrp); |
| ret = PTR_ERR_OR_ZERO(task); |
| if (ret) |
| goto out_unlock; |
| @@ -519,6 +520,7 @@ static ssize_t __cgroup1_procs_write(struct kernfs_open_file *of, |
| goto out_finish; |
| |
| ret = cgroup_attach_task(cgrp, task, threadgroup); |
| + trace_android_vh_cgroup_set_task(ret, task); |
| |
| out_finish: |
| cgroup_procs_write_finish(task, locked); |
| diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c |
| --- a/kernel/cgroup/cgroup.c |
| +++ b/kernel/cgroup/cgroup.c |
| @@ -61,6 +61,9 @@ |
| |
| #define CREATE_TRACE_POINTS |
| #include <trace/events/cgroup.h> |
| +#undef CREATE_TRACE_POINTS |
| + |
| +#include <trace/hooks/cgroup.h> |
| |
| #define CGROUP_FILE_NAME_MAX (MAX_CGROUP_TYPE_NAMELEN + \ |
| MAX_CFTYPE_NAME + 2) |
| @@ -2512,6 +2515,7 @@ static int cgroup_migrate_execute(struct cgroup_mgctx *mgctx) |
| do_each_subsys_mask(ss, ssid, mgctx->ss_mask) { |
| if (ss->attach) { |
| tset->ssid = ssid; |
| + trace_android_vh_cgroup_attach(ss, tset); |
| ss->attach(tset); |
| } |
| } while_each_subsys_mask(); |
| @@ -2812,11 +2816,13 @@ int cgroup_attach_task(struct cgroup *dst_cgrp, struct task_struct *leader, |
| } |
| |
| struct task_struct *cgroup_procs_write_start(char *buf, bool threadgroup, |
| - bool *locked) |
| + bool *locked, |
| + struct cgroup *dst_cgrp) |
| __acquires(&cgroup_threadgroup_rwsem) |
| { |
| struct task_struct *tsk; |
| pid_t pid; |
| + bool force_migration = false; |
| |
| if (kstrtoint(strstrip(buf), 0, &pid) || pid < 0) |
| return ERR_PTR(-EINVAL); |
| @@ -2851,13 +2857,16 @@ struct task_struct *cgroup_procs_write_start(char *buf, bool threadgroup, |
| if (threadgroup) |
| tsk = tsk->group_leader; |
| |
| + if (tsk->flags & PF_KTHREAD) |
| + trace_android_rvh_cgroup_force_kthread_migration(tsk, dst_cgrp, &force_migration); |
| + |
| /* |
| * kthreads may acquire PF_NO_SETAFFINITY during initialization. |
| * If userland migrates such a kthread to a non-root cgroup, it can |
| * become trapped in a cpuset, or RT kthread may be born in a |
| * cgroup with no rt_runtime allocated. Just say no. |
| */ |
| - if (tsk->no_cgroup_migration || (tsk->flags & PF_NO_SETAFFINITY)) { |
| + if (!force_migration && (tsk->no_cgroup_migration || (tsk->flags & PF_NO_SETAFFINITY))) { |
| tsk = ERR_PTR(-EINVAL); |
| goto out_unlock_threadgroup; |
| } |
| @@ -4899,7 +4908,7 @@ static ssize_t __cgroup_procs_write(struct kernfs_open_file *of, char *buf, |
| if (!dst_cgrp) |
| return -ENODEV; |
| |
| - task = cgroup_procs_write_start(buf, threadgroup, &locked); |
| + task = cgroup_procs_write_start(buf, threadgroup, &locked, dst_cgrp); |
| ret = PTR_ERR_OR_ZERO(task); |
| if (ret) |
| goto out_unlock; |
| diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c |
| --- a/kernel/cgroup/cpuset.c |
| +++ b/kernel/cgroup/cpuset.c |
| @@ -66,6 +66,8 @@ |
| #include <linux/cgroup.h> |
| #include <linux/wait.h> |
| |
| +#include <trace/hooks/sched.h> |
| + |
| DEFINE_STATIC_KEY_FALSE(cpusets_pre_enable_key); |
| DEFINE_STATIC_KEY_FALSE(cpusets_enabled_key); |
| |
| @@ -1090,6 +1092,18 @@ void rebuild_sched_domains(void) |
| cpus_read_unlock(); |
| } |
| |
| +static int update_cpus_allowed(struct cpuset *cs, struct task_struct *p, |
| + const struct cpumask *new_mask) |
| +{ |
| + int ret = -EINVAL; |
| + |
| + trace_android_rvh_update_cpus_allowed(p, cs->cpus_requested, new_mask, &ret); |
| + if (!ret) |
| + return ret; |
| + |
| + return set_cpus_allowed_ptr(p, new_mask); |
| +} |
| + |
| /** |
| * update_tasks_cpumask - Update the cpumasks of tasks in the cpuset. |
| * @cs: the cpuset in which each task's cpus_allowed mask needs to be changed |
| @@ -1105,7 +1119,7 @@ static void update_tasks_cpumask(struct cpuset *cs) |
| |
| css_task_iter_start(&cs->css, 0, &it); |
| while ((task = css_task_iter_next(&it))) |
| - set_cpus_allowed_ptr(task, cs->effective_cpus); |
| + update_cpus_allowed(cs, task, cs->effective_cpus); |
| css_task_iter_end(&it); |
| } |
| |
| @@ -2272,7 +2286,7 @@ static void cpuset_attach(struct cgroup_taskset *tset) |
| * can_attach beforehand should guarantee that this doesn't |
| * fail. TODO: have a better way to handle failure here |
| */ |
| - WARN_ON_ONCE(set_cpus_allowed_ptr(task, cpus_attach)); |
| + WARN_ON_ONCE(update_cpus_allowed(cs, task, cpus_attach)); |
| |
| cpuset_change_task_nodemask(task, &cpuset_attach_nodemask_to); |
| cpuset_update_task_spread_flag(cs, task); |
| diff --git a/kernel/cred.c b/kernel/cred.c |
| --- a/kernel/cred.c |
| +++ b/kernel/cred.c |
| @@ -17,6 +17,8 @@ |
| #include <linux/cn_proc.h> |
| #include <linux/uidgid.h> |
| |
| +#include <trace/hooks/creds.h> |
| + |
| #if 0 |
| #define kdebug(FMT, ...) \ |
| printk("[%-5.5s%5u] " FMT "\n", \ |
| @@ -181,6 +183,7 @@ void exit_creds(struct task_struct *tsk) |
| key_put(tsk->cached_requested_key); |
| tsk->cached_requested_key = NULL; |
| #endif |
| + trace_android_rvh_exit_creds(tsk, cred); |
| } |
| |
| /** |
| @@ -499,6 +502,7 @@ int commit_creds(struct cred *new) |
| inc_rlimit_ucounts(new->ucounts, UCOUNT_RLIMIT_NPROC, 1); |
| rcu_assign_pointer(task->real_cred, new); |
| rcu_assign_pointer(task->cred, new); |
| + trace_android_rvh_commit_creds(task, new); |
| if (new->user != old->user || new->user_ns != old->user_ns) |
| dec_rlimit_ucounts(old->ucounts, UCOUNT_RLIMIT_NPROC, 1); |
| alter_cred_subscribers(old, -2); |
| @@ -576,6 +580,7 @@ const struct cred *override_creds(const struct cred *new) |
| get_new_cred((struct cred *)new); |
| alter_cred_subscribers(new, 1); |
| rcu_assign_pointer(current->cred, new); |
| + trace_android_rvh_override_creds(current, new); |
| alter_cred_subscribers(old, -1); |
| |
| kdebug("override_creds() = %p{%d,%d}", old, |
| @@ -604,6 +609,7 @@ void revert_creds(const struct cred *old) |
| validate_creds(override); |
| alter_cred_subscribers(old, 1); |
| rcu_assign_pointer(current->cred, old); |
| + trace_android_rvh_revert_creds(current, old); |
| alter_cred_subscribers(override, -1); |
| put_cred(override); |
| } |
| diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c |
| --- a/kernel/locking/rwsem.c |
| +++ b/kernel/locking/rwsem.c |
| @@ -30,7 +30,6 @@ |
| |
| #ifndef CONFIG_PREEMPT_RT |
| #include "lock_events.h" |
| -#include <trace/hooks/dtask.h> |
| |
| /* |
| * The least significant 2 bits of the owner value has the following |
| @@ -1025,7 +1024,6 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat |
| wake_up_q(&wake_q); |
| |
| /* wait to be given the lock */ |
| - trace_android_vh_rwsem_read_wait_start(sem); |
| for (;;) { |
| set_current_state(state); |
| if (!smp_load_acquire(&waiter.task)) { |
| @@ -1045,7 +1043,6 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat |
| } |
| |
| __set_current_state(TASK_RUNNING); |
| - trace_android_vh_rwsem_read_wait_finish(sem); |
| lockevent_inc(rwsem_rlock); |
| return sem; |
| |
| @@ -1053,7 +1050,6 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat |
| rwsem_del_waiter(sem, &waiter); |
| raw_spin_unlock_irq(&sem->wait_lock); |
| __set_current_state(TASK_RUNNING); |
| - trace_android_vh_rwsem_read_wait_finish(sem); |
| lockevent_inc(rwsem_rlock_fail); |
| return ERR_PTR(-EINTR); |
| } |
| @@ -1121,7 +1117,6 @@ rwsem_down_write_slowpath(struct rw_semaphore *sem, int state) |
| |
| wait: |
| /* wait until we successfully acquire the lock */ |
| - trace_android_vh_rwsem_write_wait_start(sem); |
| set_current_state(state); |
| for (;;) { |
| if (rwsem_try_write_lock(sem, &waiter)) { |
| @@ -1160,14 +1155,12 @@ rwsem_down_write_slowpath(struct rw_semaphore *sem, int state) |
| raw_spin_lock_irq(&sem->wait_lock); |
| } |
| __set_current_state(TASK_RUNNING); |
| - trace_android_vh_rwsem_write_wait_finish(sem); |
| raw_spin_unlock_irq(&sem->wait_lock); |
| lockevent_inc(rwsem_wlock); |
| return sem; |
| |
| out_nolock: |
| __set_current_state(TASK_RUNNING); |
| - trace_android_vh_rwsem_write_wait_finish(sem); |
| raw_spin_lock_irq(&sem->wait_lock); |
| rwsem_del_waiter(sem, &waiter); |
| if (!list_empty(&sem->wait_list)) |
| diff --git a/kernel/module.c b/kernel/module.c |
| --- a/kernel/module.c |
| +++ b/kernel/module.c |
| @@ -63,6 +63,10 @@ |
| #define CREATE_TRACE_POINTS |
| #include <trace/events/module.h> |
| |
| +#undef CREATE_TRACE_POINTS |
| +#include <trace/hooks/module.h> |
| +#include <trace/hooks/memory.h> |
| + |
| #ifndef ARCH_SHF_SMALL |
| #define ARCH_SHF_SMALL 0 |
| #endif |
| @@ -2202,6 +2206,10 @@ static void free_module(struct module *mod) |
| |
| /* This may be empty, but that's OK */ |
| module_arch_freeing_init(mod); |
| + trace_android_vh_set_memory_rw((unsigned long)mod->init_layout.base, |
| + (mod->init_layout.size)>>PAGE_SHIFT); |
| + trace_android_vh_set_memory_nx((unsigned long)mod->init_layout.base, |
| + (mod->init_layout.size)>>PAGE_SHIFT); |
| module_memfree(mod->init_layout.base); |
| kfree(mod->args); |
| percpu_modfree(mod); |
| @@ -2210,6 +2218,10 @@ static void free_module(struct module *mod) |
| lockdep_free_key_range(mod->core_layout.base, mod->core_layout.size); |
| |
| /* Finally, free the core (containing the module structure) */ |
| + trace_android_vh_set_memory_rw((unsigned long)mod->core_layout.base, |
| + (mod->core_layout.size)>>PAGE_SHIFT); |
| + trace_android_vh_set_memory_nx((unsigned long)mod->core_layout.base, |
| + (mod->core_layout.size)>>PAGE_SHIFT); |
| module_memfree(mod->core_layout.base); |
| } |
| |
| @@ -3626,7 +3638,15 @@ static void module_deallocate(struct module *mod, struct load_info *info) |
| { |
| percpu_modfree(mod); |
| module_arch_freeing_init(mod); |
| + trace_android_vh_set_memory_rw((unsigned long)mod->init_layout.base, |
| + (mod->init_layout.size)>>PAGE_SHIFT); |
| + trace_android_vh_set_memory_nx((unsigned long)mod->init_layout.base, |
| + (mod->init_layout.size)>>PAGE_SHIFT); |
| module_memfree(mod->init_layout.base); |
| + trace_android_vh_set_memory_rw((unsigned long)mod->core_layout.base, |
| + (mod->core_layout.size)>>PAGE_SHIFT); |
| + trace_android_vh_set_memory_nx((unsigned long)mod->core_layout.base, |
| + (mod->core_layout.size)>>PAGE_SHIFT); |
| module_memfree(mod->core_layout.base); |
| } |
| |
| @@ -3784,8 +3804,13 @@ static noinline int do_init_module(struct module *mod) |
| rcu_assign_pointer(mod->kallsyms, &mod->core_kallsyms); |
| #endif |
| module_enable_ro(mod, true); |
| + trace_android_vh_set_module_permit_after_init(mod); |
| mod_tree_remove_init(mod); |
| module_arch_freeing_init(mod); |
| + trace_android_vh_set_memory_rw((unsigned long)mod->init_layout.base, |
| + (mod->init_layout.size)>>PAGE_SHIFT); |
| + trace_android_vh_set_memory_nx((unsigned long)mod->init_layout.base, |
| + (mod->init_layout.size)>>PAGE_SHIFT); |
| mod->init_layout.base = NULL; |
| mod->init_layout.size = 0; |
| mod->init_layout.ro_size = 0; |
| @@ -3896,6 +3921,7 @@ static int complete_formation(struct module *mod, struct load_info *info) |
| module_enable_ro(mod, false); |
| module_enable_nx(mod); |
| module_enable_x(mod); |
| + trace_android_vh_set_module_permit_before_init(mod); |
| |
| /* |
| * Mark state as coming so strong_try_module_get() ignores us, |
| diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c |
| --- a/kernel/printk/printk.c |
| +++ b/kernel/printk/printk.c |
| @@ -54,6 +54,8 @@ |
| #include <trace/events/initcall.h> |
| #define CREATE_TRACE_POINTS |
| #include <trace/events/printk.h> |
| +#undef CREATE_TRACE_POINTS |
| +#include <trace/hooks/printk.h> |
| |
| #include "printk_ringbuffer.h" |
| #include "console_cmdline.h" |
| @@ -2501,6 +2503,12 @@ void resume_console(void) |
| */ |
| static int console_cpu_notify(unsigned int cpu) |
| { |
| + int flag = 0; |
| + |
| + trace_android_vh_printk_hotplug(&flag); |
| + if (flag) |
| + return 0; |
| + |
| if (!cpuhp_tasks_frozen) { |
| /* If trylock fails, someone else is doing the printing */ |
| if (console_trylock()) |
| diff --git a/kernel/sys.c b/kernel/sys.c |
| --- a/kernel/sys.c |
| +++ b/kernel/sys.c |
| @@ -76,6 +76,8 @@ |
| |
| #include "uid16.h" |
| |
| +#include <trace/hooks/sys.h> |
| + |
| #ifndef SET_UNALIGN_CTL |
| # define SET_UNALIGN_CTL(a, b) (-EINVAL) |
| #endif |
| @@ -2686,6 +2688,7 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, |
| error = -EINVAL; |
| break; |
| } |
| + trace_android_vh_syscall_prctl_finished(option, me); |
| return error; |
| } |
| |
| diff --git a/kernel/time/sched_clock.c b/kernel/time/sched_clock.c |
| --- a/kernel/time/sched_clock.c |
| +++ b/kernel/time/sched_clock.c |
| @@ -16,6 +16,7 @@ |
| #include <linux/sched_clock.h> |
| #include <linux/seqlock.h> |
| #include <linux/bitops.h> |
| +#include <trace/hooks/epoch.h> |
| |
| #include "timekeeping.h" |
| |
| @@ -270,6 +271,7 @@ int sched_clock_suspend(void) |
| update_sched_clock(); |
| hrtimer_cancel(&sched_clock_timer); |
| rd->read_sched_clock = suspended_sched_clock_read; |
| + trace_android_vh_show_suspend_epoch_val(rd->epoch_ns, rd->epoch_cyc); |
| |
| return 0; |
| } |
| @@ -281,6 +283,7 @@ void sched_clock_resume(void) |
| rd->epoch_cyc = cd.actual_read_sched_clock(); |
| hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL_HARD); |
| rd->read_sched_clock = cd.actual_read_sched_clock; |
| + trace_android_vh_show_resume_epoch_val(rd->epoch_cyc); |
| } |
| |
| static struct syscore_ops sched_clock_ops = { |
| diff --git a/kernel/time/tick-common.c b/kernel/time/tick-common.c |
| --- a/kernel/time/tick-common.c |
| +++ b/kernel/time/tick-common.c |
| @@ -17,6 +17,7 @@ |
| #include <linux/sched.h> |
| #include <linux/module.h> |
| #include <trace/events/power.h> |
| +#include <trace/hooks/sched.h> |
| |
| #include <asm/irq_regs.h> |
| |
| @@ -95,6 +96,7 @@ static void tick_periodic(int cpu) |
| write_seqcount_end(&jiffies_seq); |
| raw_spin_unlock(&jiffies_lock); |
| update_wall_time(); |
| + trace_android_vh_jiffies_update(NULL); |
| } |
| |
| update_process_times(user_mode(get_irq_regs())); |
| diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c |
| --- a/kernel/time/tick-sched.c |
| +++ b/kernel/time/tick-sched.c |
| @@ -26,6 +26,7 @@ |
| #include <linux/posix-timers.h> |
| #include <linux/context_tracking.h> |
| #include <linux/mm.h> |
| +#include <trace/hooks/sched.h> |
| |
| #include <asm/irq_regs.h> |
| |
| @@ -193,8 +194,10 @@ static void tick_sched_do_timer(struct tick_sched *ts, ktime_t now) |
| #endif |
| |
| /* Check, if the jiffies need an update */ |
| - if (tick_do_timer_cpu == cpu) |
| + if (tick_do_timer_cpu == cpu) { |
| tick_do_update_jiffies64(now); |
| + trace_android_vh_jiffies_update(NULL); |
| + } |
| |
| if (ts->inidle) |
| ts->got_idle_tick = 1; |
| diff --git a/kernel/time/timer.c b/kernel/time/timer.c |
| --- a/kernel/time/timer.c |
| +++ b/kernel/time/timer.c |
| @@ -55,6 +55,8 @@ |
| |
| #define CREATE_TRACE_POINTS |
| #include <trace/events/timer.h> |
| +#undef CREATE_TRACE_POINTS |
| +#include <trace/hooks/timer.h> |
| |
| __visible u64 jiffies_64 __cacheline_aligned_in_smp = INITIAL_JIFFIES; |
| |
| @@ -503,6 +505,7 @@ static inline unsigned calc_index(unsigned long expires, unsigned lvl, |
| * Round up with level granularity to prevent this. |
| */ |
| expires = (expires + LVL_GRAN(lvl)) >> LVL_SHIFT(lvl); |
| + trace_android_vh_timer_calc_index(lvl, &expires); |
| *bucket_expiry = expires << LVL_SHIFT(lvl); |
| return LVL_OFFS(lvl) + (expires & LVL_MASK); |
| } |
| diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c |
| --- a/kernel/trace/trace.c |
| +++ b/kernel/trace/trace.c |
| @@ -49,6 +49,7 @@ |
| #include <linux/fsnotify.h> |
| #include <linux/irq_work.h> |
| #include <linux/workqueue.h> |
| +#include <trace/hooks/ftrace_dump.h> |
| |
| #include "trace.h" |
| #include "trace_output.h" |
| @@ -9713,8 +9714,17 @@ fs_initcall(tracer_init_tracefs); |
| static int trace_panic_handler(struct notifier_block *this, |
| unsigned long event, void *unused) |
| { |
| + bool ftrace_check = false; |
| + |
| + trace_android_vh_ftrace_oops_enter(&ftrace_check); |
| + |
| + if (ftrace_check) |
| + return NOTIFY_OK; |
| + |
| if (ftrace_dump_on_oops) |
| ftrace_dump(ftrace_dump_on_oops); |
| + |
| + trace_android_vh_ftrace_oops_exit(&ftrace_check); |
| return NOTIFY_OK; |
| } |
| |
| @@ -9728,6 +9738,13 @@ static int trace_die_handler(struct notifier_block *self, |
| unsigned long val, |
| void *data) |
| { |
| + bool ftrace_check = false; |
| + |
| + trace_android_vh_ftrace_oops_enter(&ftrace_check); |
| + |
| + if (ftrace_check) |
| + return NOTIFY_OK; |
| + |
| switch (val) { |
| case DIE_OOPS: |
| if (ftrace_dump_on_oops) |
| @@ -9736,6 +9753,8 @@ static int trace_die_handler(struct notifier_block *self, |
| default: |
| break; |
| } |
| + |
| + trace_android_vh_ftrace_oops_exit(&ftrace_check); |
| return NOTIFY_OK; |
| } |
| |
| @@ -9760,6 +9779,8 @@ static struct notifier_block trace_die_notifier = { |
| void |
| trace_printk_seq(struct trace_seq *s) |
| { |
| + bool dump_printk = true; |
| + |
| /* Probably should print a warning here. */ |
| if (s->seq.len >= TRACE_MAX_PRINT) |
| s->seq.len = TRACE_MAX_PRINT; |
| @@ -9775,7 +9796,9 @@ trace_printk_seq(struct trace_seq *s) |
| /* should be zero ended, but we are paranoid. */ |
| s->buffer[s->seq.len] = 0; |
| |
| - printk(KERN_TRACE "%s", s->buffer); |
| + trace_android_vh_ftrace_dump_buffer(s, &dump_printk); |
| + if (dump_printk) |
| + printk(KERN_TRACE "%s", s->buffer); |
| |
| trace_seq_init(s); |
| } |
| @@ -9808,6 +9831,8 @@ void ftrace_dump(enum ftrace_dump_mode oops_dump_mode) |
| unsigned int old_userobj; |
| unsigned long flags; |
| int cnt = 0, cpu; |
| + bool ftrace_check = false; |
| + unsigned long size; |
| |
| /* Only allow one dump user at a time. */ |
| if (atomic_inc_return(&dump_running) != 1) { |
| @@ -9837,6 +9862,8 @@ void ftrace_dump(enum ftrace_dump_mode oops_dump_mode) |
| |
| for_each_tracing_cpu(cpu) { |
| atomic_inc(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled); |
| + size = ring_buffer_size(iter.array_buffer->buffer, cpu); |
| + trace_android_vh_ftrace_size_check(size, &ftrace_check); |
| } |
| |
| old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ; |
| @@ -9844,6 +9871,9 @@ void ftrace_dump(enum ftrace_dump_mode oops_dump_mode) |
| /* don't look at user memory in panic mode */ |
| tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ; |
| |
| + if (ftrace_check) |
| + goto out_enable; |
| + |
| switch (oops_dump_mode) { |
| case DUMP_ALL: |
| iter.cpu_file = RING_BUFFER_ALL_CPUS; |
| @@ -9874,6 +9904,7 @@ void ftrace_dump(enum ftrace_dump_mode oops_dump_mode) |
| */ |
| |
| while (!trace_empty(&iter)) { |
| + ftrace_check = true; |
| |
| if (!cnt) |
| printk(KERN_TRACE "---------------------------------\n"); |
| @@ -9881,7 +9912,9 @@ void ftrace_dump(enum ftrace_dump_mode oops_dump_mode) |
| cnt++; |
| |
| trace_iterator_reset(&iter); |
| - iter.iter_flags |= TRACE_FILE_LAT_FMT; |
| + trace_android_vh_ftrace_format_check(&ftrace_check); |
| + if (ftrace_check) |
| + iter.iter_flags |= TRACE_FILE_LAT_FMT; |
| |
| if (trace_find_next_entry_inc(&iter) != NULL) { |
| int ret; |
| diff --git a/kernel/trace/trace_preemptirq.c b/kernel/trace/trace_preemptirq.c |
| --- a/kernel/trace/trace_preemptirq.c |
| +++ b/kernel/trace/trace_preemptirq.c |
| @@ -14,6 +14,8 @@ |
| |
| #define CREATE_TRACE_POINTS |
| #include <trace/events/preemptirq.h> |
| +#undef CREATE_TRACE_POINTS |
| +#include <trace/hooks/preemptirq.h> |
| |
| #ifdef CONFIG_TRACE_IRQFLAGS |
| /* Per-cpu variable to prevent redundant calls when IRQs already off */ |
| @@ -28,8 +30,11 @@ static DEFINE_PER_CPU(int, tracing_irq_cpu); |
| void trace_hardirqs_on_prepare(void) |
| { |
| if (this_cpu_read(tracing_irq_cpu)) { |
| - if (!in_nmi()) |
| + if (!in_nmi()) { |
| trace_irq_enable(CALLER_ADDR0, CALLER_ADDR1); |
| + trace_android_rvh_irqs_enable(CALLER_ADDR0, |
| + CALLER_ADDR1); |
| + } |
| tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1); |
| this_cpu_write(tracing_irq_cpu, 0); |
| } |
| @@ -40,8 +45,11 @@ NOKPROBE_SYMBOL(trace_hardirqs_on_prepare); |
| void trace_hardirqs_on(void) |
| { |
| if (this_cpu_read(tracing_irq_cpu)) { |
| - if (!in_nmi()) |
| + if (!in_nmi()) { |
| trace_irq_enable_rcuidle(CALLER_ADDR0, CALLER_ADDR1); |
| + trace_android_rvh_irqs_enable(CALLER_ADDR0, |
| + CALLER_ADDR1); |
| + } |
| tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1); |
| this_cpu_write(tracing_irq_cpu, 0); |
| } |
| @@ -63,8 +71,11 @@ void trace_hardirqs_off_finish(void) |
| if (!this_cpu_read(tracing_irq_cpu)) { |
| this_cpu_write(tracing_irq_cpu, 1); |
| tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1); |
| - if (!in_nmi()) |
| + if (!in_nmi()) { |
| trace_irq_disable(CALLER_ADDR0, CALLER_ADDR1); |
| + trace_android_rvh_irqs_disable(CALLER_ADDR0, |
| + CALLER_ADDR1); |
| + } |
| } |
| |
| } |
| @@ -78,8 +89,11 @@ void trace_hardirqs_off(void) |
| if (!this_cpu_read(tracing_irq_cpu)) { |
| this_cpu_write(tracing_irq_cpu, 1); |
| tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1); |
| - if (!in_nmi()) |
| + if (!in_nmi()) { |
| trace_irq_disable_rcuidle(CALLER_ADDR0, CALLER_ADDR1); |
| + trace_android_rvh_irqs_disable(CALLER_ADDR0, |
| + CALLER_ADDR1); |
| + } |
| } |
| } |
| EXPORT_SYMBOL(trace_hardirqs_off); |
| @@ -88,8 +102,11 @@ NOKPROBE_SYMBOL(trace_hardirqs_off); |
| __visible void trace_hardirqs_on_caller(unsigned long caller_addr) |
| { |
| if (this_cpu_read(tracing_irq_cpu)) { |
| - if (!in_nmi()) |
| + if (!in_nmi()) { |
| trace_irq_enable_rcuidle(CALLER_ADDR0, caller_addr); |
| + trace_android_rvh_irqs_enable(CALLER_ADDR0, |
| + caller_addr); |
| + } |
| tracer_hardirqs_on(CALLER_ADDR0, caller_addr); |
| this_cpu_write(tracing_irq_cpu, 0); |
| } |
| @@ -107,8 +124,11 @@ __visible void trace_hardirqs_off_caller(unsigned long caller_addr) |
| if (!this_cpu_read(tracing_irq_cpu)) { |
| this_cpu_write(tracing_irq_cpu, 1); |
| tracer_hardirqs_off(CALLER_ADDR0, caller_addr); |
| - if (!in_nmi()) |
| + if (!in_nmi()) { |
| trace_irq_disable_rcuidle(CALLER_ADDR0, caller_addr); |
| + trace_android_rvh_irqs_enable(CALLER_ADDR0, |
| + caller_addr); |
| + } |
| } |
| } |
| EXPORT_SYMBOL(trace_hardirqs_off_caller); |
| @@ -119,15 +139,19 @@ NOKPROBE_SYMBOL(trace_hardirqs_off_caller); |
| |
| void trace_preempt_on(unsigned long a0, unsigned long a1) |
| { |
| - if (!in_nmi()) |
| + if (!in_nmi()) { |
| trace_preempt_enable_rcuidle(a0, a1); |
| + trace_android_rvh_preempt_enable(a0, a1); |
| + } |
| tracer_preempt_on(a0, a1); |
| } |
| |
| void trace_preempt_off(unsigned long a0, unsigned long a1) |
| { |
| - if (!in_nmi()) |
| + if (!in_nmi()) { |
| trace_preempt_disable_rcuidle(a0, a1); |
| + trace_android_rvh_preempt_disable(a0, a1); |
| + } |
| tracer_preempt_off(a0, a1); |
| } |
| #endif |
| diff --git a/kernel/workqueue.c b/kernel/workqueue.c |
| --- a/kernel/workqueue.c |
| +++ b/kernel/workqueue.c |
| @@ -54,6 +54,10 @@ |
| |
| #include "workqueue_internal.h" |
| |
| +#include <trace/hooks/wqlockup.h> |
| +/* events/workqueue.h uses default TRACE_INCLUDE_PATH */ |
| +#undef TRACE_INCLUDE_PATH |
| + |
| enum { |
| /* |
| * worker_pool flags |
| @@ -5890,6 +5894,7 @@ static void wq_watchdog_timer_fn(struct timer_list *unused) |
| pr_cont_pool_info(pool); |
| pr_cont(" stuck for %us!\n", |
| jiffies_to_msecs(now - pool_ts) / 1000); |
| + trace_android_vh_wq_lockup_pool(pool->cpu, pool_ts); |
| } |
| } |
| |