blob: 85a3268fb78ed4c5443fb9a9579fa2dbf677b967 [file] [log] [blame]
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: JianMin Liu <jian-min.liu@mediatek.com>
Date: Tue, 21 Jul 2020 13:36:22 +0800
Subject: NOUPSTREAM: ANDROID: kernel: Core kernel related vendor hooks
- Add the hook to apply vendor's performance tune for owner
of rwsem.
- Add the hook for the waiter list of rwsem to allow
vendor perform waiting queue enhancement
- ANDROID_VENDOR_DATA added to rw_semaphore
[CPNOTE: 27/05/21] Lee: Vendor related code - maintain forever
Squash:
NOUPSTREAM: ANDROID: workqueue: add vendor hook for wq lockup information
NOUPSTREAM: ANDROID: printk: add vendor hook for console flush
NOUPSTREAM: ANDROID: vendor_hooks: Add hook for jiffies updates
NOUPSTREAM: ANDROID: sched_clock: Add vendor hook to print epoch values
NOUPSTREAM: ANDROID: sched/cpuset: Add vendor hook to change tasks affinity
NOUPSTREAM: ANDROID: tracing: Add restricted vendor hooks for preemptirq events
NOUPSTREAM: ANDROID: ftrace: vendor hook for ftrace dump on oops
NOUPSTREAM: ANDROID: cgroup: Add vendor hook to the cgroup
NOUPSTREAM: ANDROID: Add vendor hooks when syscall prctl finished
NOUPSTREAM: ANDROID: timer: Add vendor hook for timer calc index
ANDROID: cgroup: Add android_rvh_cgroup_force_kthread_migration
ANDROID: kernel: Add restricted vendor hook in creds
Bug: 161400830
Bug: 169374262
Bug: 165340180
Bug: 148928265
Bug: 172945021
Bug: 174125747
Bug: 174541725
Bug: 177387994
Bug: 181917687
Bug: 181819699
Bug: 178758017
Bug: 184594949
Bug: 181639260
Signed-off-by: JianMin Liu <jian-min.liu@mediatek.com>
Change-Id: I007a5e26f3db2adaeaf4e5ccea414ce7abfa83b8
ANDROID: module: Add vendor hook
Add vendor hook for module init, so we can get memory type and
use it to do memory type check for architecture
dependent page table setting.
Bug: 181639260
Signed-off-by: Kuan-Ying Lee <Kuan-Ying.Lee@mediatek.com>
Change-Id: I95b70d7a57994f2548fddfb2290d4c9136f58785
ANDROID: syscall_check: add vendor hook for bpf syscall
Through this vendor hook, we can get the timing to check
current running task for the validation of its credential
and bpf operations.
Bug: 191291287
Signed-off-by: Kuan-Ying Lee <Kuan-Ying.Lee@mediatek.com>
Change-Id: Ie4ed8df7ad66df2486fc7e52a26d9191fc0c176e
(cherry picked from commit 38abaebab795d87a4e5bbf2ce00e7e37a3db3847)
---
drivers/android/vendor_hooks.c | 48 ++++++++++++++++++++++++---
include/trace/hooks/cgroup.h | 26 +++++++++++++++
include/trace/hooks/creds.h | 34 ++++++++++++++++++++
include/trace/hooks/dtask.h | 14 --------
include/trace/hooks/epoch.h | 23 +++++++++++++
include/trace/hooks/ftrace_dump.h | 50 +++++++++++++++++++++++++++++
include/trace/hooks/memory.h | 24 ++++++++++++++
include/trace/hooks/module.h | 25 +++++++++++++++
include/trace/hooks/preemptirq.h | 32 ++++++++++++++++++
include/trace/hooks/printk.h | 19 +++++++++++
include/trace/hooks/sched.h | 9 ++++++
include/trace/hooks/sys.h | 17 ++++++++++
include/trace/hooks/syscall_check.h | 5 +++
include/trace/hooks/timer.h | 19 +++++++++++
include/trace/hooks/wqlockup.h | 20 ++++++++++++
kernel/bpf/syscall.c | 4 +++
kernel/cgroup/cgroup-internal.h | 3 +-
kernel/cgroup/cgroup-v1.c | 4 ++-
kernel/cgroup/cgroup.c | 15 +++++++--
kernel/cgroup/cpuset.c | 18 +++++++++--
kernel/cred.c | 6 ++++
kernel/locking/rwsem.c | 7 ----
kernel/module.c | 26 +++++++++++++++
kernel/printk/printk.c | 8 +++++
kernel/sys.c | 3 ++
kernel/time/sched_clock.c | 3 ++
kernel/time/tick-common.c | 2 ++
kernel/time/tick-sched.c | 5 ++-
kernel/time/timer.c | 3 ++
kernel/trace/trace.c | 37 +++++++++++++++++++--
kernel/trace/trace_preemptirq.c | 40 ++++++++++++++++++-----
kernel/workqueue.c | 5 +++
32 files changed, 511 insertions(+), 43 deletions(-)
create mode 100644 include/trace/hooks/cgroup.h
create mode 100644 include/trace/hooks/creds.h
create mode 100644 include/trace/hooks/epoch.h
create mode 100644 include/trace/hooks/ftrace_dump.h
create mode 100644 include/trace/hooks/memory.h
create mode 100644 include/trace/hooks/module.h
create mode 100644 include/trace/hooks/preemptirq.h
create mode 100644 include/trace/hooks/printk.h
create mode 100644 include/trace/hooks/sys.h
create mode 100644 include/trace/hooks/timer.h
create mode 100644 include/trace/hooks/wqlockup.h
diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c
--- a/drivers/android/vendor_hooks.c
+++ b/drivers/android/vendor_hooks.c
@@ -12,10 +12,22 @@
#include <trace/hooks/fpsimd.h>
#include <trace/hooks/dtask.h>
#include <trace/hooks/mpam.h>
+#include <trace/hooks/wqlockup.h>
#include <trace/hooks/debug.h>
+#include <trace/hooks/printk.h>
+#include <trace/hooks/epoch.h>
#include <trace/hooks/mm.h>
+#include <trace/hooks/preemptirq.h>
+#include <trace/hooks/ftrace_dump.h>
+#include <trace/hooks/cgroup.h>
+#include <trace/hooks/sys.h>
#include <trace/hooks/iommu.h>
+#include <trace/hooks/net.h>
+#include <trace/hooks/timer.h>
#include <trace/hooks/vmscan.h>
+#include <trace/hooks/creds.h>
+#include <trace/hooks/memory.h>
+#include <trace/hooks/module.h>
#include <trace/hooks/syscall_check.h>
/*
@@ -36,14 +48,16 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_finish_prio_fork);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_user_nice);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_setscheduler);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_is_fpsimd_save);
-EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_read_wait_start);
-EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_read_wait_finish);
-EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_write_wait_start);
-EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_write_wait_finish);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_sched_show_task);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mpam_set);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_find_busiest_group);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_wq_lockup_pool);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ipi_stop);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_dump_throttled_rt_tasks);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_printk_hotplug);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_jiffies_update);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_show_suspend_epoch_val);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_show_resume_epoch_val);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_sched_newidle_balance);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_sched_nohz_balancer_kick);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_find_busiest_queue);
@@ -53,9 +67,14 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_set_sugov_sched_attr);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_iowait);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_sugov_update);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_sched_setaffinity);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_update_cpus_allowed);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_skip_swapcache_flags);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_gfp_zone_flags);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_readahead_gfp_mask);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_preempt_disable);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_preempt_enable);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_irqs_disable);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_irqs_enable);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_task_cpu);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_try_to_wake_up);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_try_to_wake_up_success);
@@ -81,7 +100,19 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_schedule_bug);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_sched_exec);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_map_util_freq);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_em_cpu_energy);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ftrace_oops_enter);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ftrace_oops_exit);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ftrace_size_check);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ftrace_format_check);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ftrace_dump_buffer);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cgroup_set_task);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_syscall_prctl_finished);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cgroup_attach);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_iommu_setup_dma_ops);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ptype_head);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_kfree_skb);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_timer_calc_index);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_cgroup_force_kthread_migration);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_check_preempt_tick);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_check_preempt_wakeup_ignore);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_replace_next_task_fair);
@@ -98,4 +129,13 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_dequeue_entity);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_entity_tick);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_enqueue_task_fair);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_dequeue_task_fair);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_commit_creds);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_exit_creds);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_override_creds);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_revert_creds);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_set_memory_nx);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_set_memory_rw);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_set_module_permit_before_init);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_set_module_permit_after_init);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_check_mmap_file);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_check_bpf_syscall);
diff --git a/include/trace/hooks/cgroup.h b/include/trace/hooks/cgroup.h
new file mode 100644
--- /dev/null
+++ b/include/trace/hooks/cgroup.h
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM cgroup
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH trace/hooks
+#if !defined(_TRACE_HOOK_CGROUP_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_CGROUP_H
+#include <linux/tracepoint.h>
+#include <trace/hooks/vendor_hooks.h>
+
+struct task_struct;
+DECLARE_HOOK(android_vh_cgroup_set_task,
+ TP_PROTO(int ret, struct task_struct *task),
+ TP_ARGS(ret, task));
+
+struct cgroup_subsys;
+struct cgroup_taskset;
+DECLARE_HOOK(android_vh_cgroup_attach,
+ TP_PROTO(struct cgroup_subsys *ss, struct cgroup_taskset *tset),
+ TP_ARGS(ss, tset))
+DECLARE_RESTRICTED_HOOK(android_rvh_cgroup_force_kthread_migration,
+ TP_PROTO(struct task_struct *tsk, struct cgroup *dst_cgrp, bool *force_migration),
+ TP_ARGS(tsk, dst_cgrp, force_migration), 1);
+#endif
+
+#include <trace/define_trace.h>
diff --git a/include/trace/hooks/creds.h b/include/trace/hooks/creds.h
new file mode 100644
--- /dev/null
+++ b/include/trace/hooks/creds.h
@@ -0,0 +1,34 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM creds
+
+#define TRACE_INCLUDE_PATH trace/hooks
+#if !defined(_TRACE_HOOK_CREDS_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_CREDS_H
+#include <linux/tracepoint.h>
+#include <trace/hooks/vendor_hooks.h>
+/*
+ * Following tracepoints are not exported in tracefs and provide a
+ * mechanism for vendor modules to hook and extend functionality
+ */
+struct cred;
+struct task_struct;
+DECLARE_RESTRICTED_HOOK(android_rvh_commit_creds,
+ TP_PROTO(const struct task_struct *task, const struct cred *new),
+ TP_ARGS(task, new), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_exit_creds,
+ TP_PROTO(const struct task_struct *task, const struct cred *cred),
+ TP_ARGS(task, cred), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_override_creds,
+ TP_PROTO(const struct task_struct *task, const struct cred *new),
+ TP_ARGS(task, new), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_revert_creds,
+ TP_PROTO(const struct task_struct *task, const struct cred *old),
+ TP_ARGS(task, old), 1);
+
+#endif /* _TRACE_HOOK_CREDS_H */
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/trace/hooks/dtask.h b/include/trace/hooks/dtask.h
--- a/include/trace/hooks/dtask.h
+++ b/include/trace/hooks/dtask.h
@@ -11,20 +11,6 @@
* Following tracepoints are not exported in tracefs and provide a
* mechanism for vendor modules to hook and extend functionality
*/
-struct rw_semaphore;
-DECLARE_HOOK(android_vh_rwsem_read_wait_start,
- TP_PROTO(struct rw_semaphore *sem),
- TP_ARGS(sem));
-DECLARE_HOOK(android_vh_rwsem_read_wait_finish,
- TP_PROTO(struct rw_semaphore *sem),
- TP_ARGS(sem));
-DECLARE_HOOK(android_vh_rwsem_write_wait_start,
- TP_PROTO(struct rw_semaphore *sem),
- TP_ARGS(sem));
-DECLARE_HOOK(android_vh_rwsem_write_wait_finish,
- TP_PROTO(struct rw_semaphore *sem),
- TP_ARGS(sem));
-
struct task_struct;
DECLARE_HOOK(android_vh_sched_show_task,
TP_PROTO(struct task_struct *task),
diff --git a/include/trace/hooks/epoch.h b/include/trace/hooks/epoch.h
new file mode 100644
--- /dev/null
+++ b/include/trace/hooks/epoch.h
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM epoch
+
+#define TRACE_INCLUDE_PATH trace/hooks
+
+#if !defined(_TRACE_HOOK_EPOCH_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_EPOCH_H
+
+#include <linux/tracepoint.h>
+#include <trace/hooks/vendor_hooks.h>
+
+DECLARE_HOOK(android_vh_show_suspend_epoch_val,
+ TP_PROTO(u64 suspend_ns, u64 suspend_cycles),
+ TP_ARGS(suspend_ns, suspend_cycles));
+
+DECLARE_HOOK(android_vh_show_resume_epoch_val,
+ TP_PROTO(u64 resume_cycles),
+ TP_ARGS(resume_cycles));
+
+#endif /* _TRACE_HOOK_EPOCH_H */
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/trace/hooks/ftrace_dump.h b/include/trace/hooks/ftrace_dump.h
new file mode 100644
--- /dev/null
+++ b/include/trace/hooks/ftrace_dump.h
@@ -0,0 +1,50 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM ftrace_dump
+
+#define TRACE_INCLUDE_PATH trace/hooks
+
+#if !defined(_TRACE_HOOK_FTRACE_DUMP_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_FTRACE_DUMP_H
+
+#include <linux/trace_seq.h>
+#include <linux/trace_events.h>
+
+#include <linux/tracepoint.h>
+#include <trace/hooks/vendor_hooks.h>
+
+#if defined(CONFIG_TRACEPOINTS) && defined(CONFIG_ANDROID_VENDOR_HOOKS)
+
+DECLARE_HOOK(android_vh_ftrace_oops_enter,
+ TP_PROTO(bool *ftrace_check),
+ TP_ARGS(ftrace_check));
+
+DECLARE_HOOK(android_vh_ftrace_oops_exit,
+ TP_PROTO(bool *ftrace_check),
+ TP_ARGS(ftrace_check));
+
+DECLARE_HOOK(android_vh_ftrace_size_check,
+ TP_PROTO(unsigned long size, bool *ftrace_check),
+ TP_ARGS(size, ftrace_check));
+
+DECLARE_HOOK(android_vh_ftrace_format_check,
+ TP_PROTO(bool *ftrace_check),
+ TP_ARGS(ftrace_check));
+
+DECLARE_HOOK(android_vh_ftrace_dump_buffer,
+ TP_PROTO(struct trace_seq *trace_buf, bool *dump_printk),
+ TP_ARGS(trace_buf, dump_printk));
+
+#else
+
+#define trace_android_vh_ftrace_oops_enter(ftrace_check)
+#define trace_android_vh_ftrace_oops_exit(ftrace_check)
+#define trace_android_vh_ftrace_size_check(size, ftrace_check)
+#define trace_android_vh_ftrace_format_check(ftrace_check)
+#define trace_android_vh_ftrace_dump_buffer(trace_buf, dump_printk)
+
+#endif
+
+#endif /* _TRACE_HOOK_FTRACE_DUMP_H */
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/trace/hooks/memory.h b/include/trace/hooks/memory.h
new file mode 100644
--- /dev/null
+++ b/include/trace/hooks/memory.h
@@ -0,0 +1,24 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM memory
+
+#define TRACE_INCLUDE_PATH trace/hooks
+#if !defined(_TRACE_HOOK_MEMORY_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_MEMORY_H
+#include <linux/tracepoint.h>
+#include <trace/hooks/vendor_hooks.h>
+/*
+ * Following tracepoints are not exported in tracefs and provide a
+ * mechanism for vendor modules to hook and extend functionality
+ */
+DECLARE_HOOK(android_vh_set_memory_nx,
+ TP_PROTO(unsigned long addr, int nr_pages),
+ TP_ARGS(addr, nr_pages));
+
+DECLARE_HOOK(android_vh_set_memory_rw,
+ TP_PROTO(unsigned long addr, int nr_pages),
+ TP_ARGS(addr, nr_pages));
+
+#endif /* _TRACE_HOOK_MEMORY_H */
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/trace/hooks/module.h b/include/trace/hooks/module.h
new file mode 100644
--- /dev/null
+++ b/include/trace/hooks/module.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM module
+
+#define TRACE_INCLUDE_PATH trace/hooks
+#if !defined(_TRACE_HOOK_MODULE_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_MODULE_H
+#include <linux/tracepoint.h>
+#include <trace/hooks/vendor_hooks.h>
+/*
+ * Following tracepoints are not exported in tracefs and provide a
+ * mechanism for vendor modules to hook and extend functionality
+ */
+struct module;
+DECLARE_HOOK(android_vh_set_module_permit_before_init,
+ TP_PROTO(const struct module *mod),
+ TP_ARGS(mod));
+
+DECLARE_HOOK(android_vh_set_module_permit_after_init,
+ TP_PROTO(const struct module *mod),
+ TP_ARGS(mod));
+
+#endif /* _TRACE_HOOK_MODULE_H */
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/trace/hooks/preemptirq.h b/include/trace/hooks/preemptirq.h
new file mode 100644
--- /dev/null
+++ b/include/trace/hooks/preemptirq.h
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM preemptirq
+
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH trace/hooks
+
+#if !defined(_TRACE_HOOK_PREEMPTIRQ_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_PREEMPTIRQ_H
+
+#include <linux/tracepoint.h>
+#include <trace/hooks/vendor_hooks.h>
+
+DECLARE_RESTRICTED_HOOK(android_rvh_preempt_disable,
+ TP_PROTO(unsigned long ip, unsigned long parent_ip),
+ TP_ARGS(ip, parent_ip), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_preempt_enable,
+ TP_PROTO(unsigned long ip, unsigned long parent_ip),
+ TP_ARGS(ip, parent_ip), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_irqs_disable,
+ TP_PROTO(unsigned long ip, unsigned long parent_ip),
+ TP_ARGS(ip, parent_ip), 1);
+
+DECLARE_RESTRICTED_HOOK(android_rvh_irqs_enable,
+ TP_PROTO(unsigned long ip, unsigned long parent_ip),
+ TP_ARGS(ip, parent_ip), 1);
+
+#endif /* _TRACE_HOOK_PREEMPTIRQ_H */
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/trace/hooks/printk.h b/include/trace/hooks/printk.h
new file mode 100644
--- /dev/null
+++ b/include/trace/hooks/printk.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM printk
+
+#define TRACE_INCLUDE_PATH trace/hooks
+
+#if !defined(_TRACE_HOOK_PRINTK_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_PRINTK_H
+
+#include <linux/tracepoint.h>
+#include <trace/hooks/vendor_hooks.h>
+
+DECLARE_HOOK(android_vh_printk_hotplug,
+ TP_PROTO(int *flag),
+ TP_ARGS(flag));
+
+#endif /* _TRACE_HOOK_PRINTK_H */
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/trace/hooks/sched.h b/include/trace/hooks/sched.h
--- a/include/trace/hooks/sched.h
+++ b/include/trace/hooks/sched.h
@@ -75,6 +75,10 @@ DECLARE_HOOK(android_vh_dump_throttled_rt_tasks,
s64 rt_period_timer_expires),
TP_ARGS(cpu, clock, rt_period, rt_runtime, rt_period_timer_expires));
+DECLARE_HOOK(android_vh_jiffies_update,
+ TP_PROTO(void *unused),
+ TP_ARGS(unused));
+
struct rq_flags;
DECLARE_RESTRICTED_HOOK(android_rvh_sched_newidle_balance,
TP_PROTO(struct rq *this_rq, struct rq_flags *rf,
@@ -115,6 +119,11 @@ DECLARE_RESTRICTED_HOOK(android_rvh_sched_setaffinity,
TP_PROTO(struct task_struct *p, const struct cpumask *in_mask, int *retval),
TP_ARGS(p, in_mask, retval), 1);
+DECLARE_RESTRICTED_HOOK(android_rvh_update_cpus_allowed,
+ TP_PROTO(struct task_struct *p, cpumask_var_t cpus_requested,
+ const struct cpumask *new_mask, int *ret),
+ TP_ARGS(p, cpus_requested, new_mask, ret), 1);
+
DECLARE_RESTRICTED_HOOK(android_rvh_set_task_cpu,
TP_PROTO(struct task_struct *p, unsigned int new_cpu),
TP_ARGS(p, new_cpu), 1);
diff --git a/include/trace/hooks/sys.h b/include/trace/hooks/sys.h
new file mode 100644
--- /dev/null
+++ b/include/trace/hooks/sys.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM sys
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH trace/hooks
+#if !defined(_TRACE_HOOK_SYS_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_SYS_H
+#include <linux/tracepoint.h>
+#include <trace/hooks/vendor_hooks.h>
+
+struct task_struct;
+DECLARE_HOOK(android_vh_syscall_prctl_finished,
+ TP_PROTO(int option, struct task_struct *task),
+ TP_ARGS(option, task));
+#endif
+
+#include <trace/define_trace.h>
diff --git a/include/trace/hooks/syscall_check.h b/include/trace/hooks/syscall_check.h
--- a/include/trace/hooks/syscall_check.h
+++ b/include/trace/hooks/syscall_check.h
@@ -12,11 +12,16 @@
* mechanism for vendor modules to hook and extend functionality
*/
struct file;
+union bpf_attr;
DECLARE_HOOK(android_vh_check_mmap_file,
TP_PROTO(const struct file *file, unsigned long prot,
unsigned long flag, unsigned long ret),
TP_ARGS(file, prot, flag, ret));
+DECLARE_HOOK(android_vh_check_bpf_syscall,
+ TP_PROTO(int cmd, const union bpf_attr *attr, unsigned int size),
+ TP_ARGS(cmd, attr, size));
+
#endif /* _TRACE_HOOK_SYSCALL_CHECK_H */
/* This part must be outside protection */
#include <trace/define_trace.h>
diff --git a/include/trace/hooks/timer.h b/include/trace/hooks/timer.h
new file mode 100644
--- /dev/null
+++ b/include/trace/hooks/timer.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM timer
+
+#define TRACE_INCLUDE_PATH trace/hooks
+
+#if !defined(_TRACE_HOOK_TIMER_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_TIMER_H
+
+#include <linux/tracepoint.h>
+#include <trace/hooks/vendor_hooks.h>
+
+DECLARE_HOOK(android_vh_timer_calc_index,
+ TP_PROTO(unsigned int lvl, unsigned long *expires),
+ TP_ARGS(lvl, expires));
+
+#endif /* _TRACE_HOOK_TIMER_H */
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/trace/hooks/wqlockup.h b/include/trace/hooks/wqlockup.h
new file mode 100644
--- /dev/null
+++ b/include/trace/hooks/wqlockup.h
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM wqlockup
+#define TRACE_INCLUDE_PATH trace/hooks
+
+#if !defined(_TRACE_HOOK_WQLOCKUP_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HOOK_WQLOCKUP_H
+#include <linux/tracepoint.h>
+#include <trace/hooks/vendor_hooks.h>
+/*
+ * Following tracepoints are not exported in tracefs and provide a
+ * mechanism for vendor modules to hook and extend functionality
+ */
+DECLARE_HOOK(android_vh_wq_lockup_pool,
+ TP_PROTO(int cpu, unsigned long pool_ts),
+ TP_ARGS(cpu, pool_ts));
+
+#endif /* _TRACE_HOOK_WQLOCKUP_H */
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -32,6 +32,8 @@
#include <linux/rcupdate_trace.h>
#include <linux/memcontrol.h>
+#include <trace/hooks/syscall_check.h>
+
#define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \
(map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \
(map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
@@ -4606,6 +4608,8 @@ static int __sys_bpf(int cmd, bpfptr_t uattr, unsigned int size)
if (copy_from_bpfptr(&attr, uattr, size) != 0)
return -EFAULT;
+ trace_android_vh_check_bpf_syscall(cmd, &attr, size);
+
err = security_bpf(cmd, &attr, size);
if (err < 0)
return err;
diff --git a/kernel/cgroup/cgroup-internal.h b/kernel/cgroup/cgroup-internal.h
--- a/kernel/cgroup/cgroup-internal.h
+++ b/kernel/cgroup/cgroup-internal.h
@@ -232,7 +232,8 @@ int cgroup_migrate(struct task_struct *leader, bool threadgroup,
int cgroup_attach_task(struct cgroup *dst_cgrp, struct task_struct *leader,
bool threadgroup);
struct task_struct *cgroup_procs_write_start(char *buf, bool threadgroup,
- bool *locked)
+ bool *locked,
+ struct cgroup *dst_cgrp);
__acquires(&cgroup_threadgroup_rwsem);
void cgroup_procs_write_finish(struct task_struct *task, bool locked)
__releases(&cgroup_threadgroup_rwsem);
diff --git a/kernel/cgroup/cgroup-v1.c b/kernel/cgroup/cgroup-v1.c
--- a/kernel/cgroup/cgroup-v1.c
+++ b/kernel/cgroup/cgroup-v1.c
@@ -17,6 +17,7 @@
#include <linux/fs_parser.h>
#include <trace/events/cgroup.h>
+#include <trace/hooks/cgroup.h>
/*
* pidlists linger the following amount before being destroyed. The goal
@@ -498,7 +499,7 @@ static ssize_t __cgroup1_procs_write(struct kernfs_open_file *of,
if (!cgrp)
return -ENODEV;
- task = cgroup_procs_write_start(buf, threadgroup, &locked);
+ task = cgroup_procs_write_start(buf, threadgroup, &locked, cgrp);
ret = PTR_ERR_OR_ZERO(task);
if (ret)
goto out_unlock;
@@ -519,6 +520,7 @@ static ssize_t __cgroup1_procs_write(struct kernfs_open_file *of,
goto out_finish;
ret = cgroup_attach_task(cgrp, task, threadgroup);
+ trace_android_vh_cgroup_set_task(ret, task);
out_finish:
cgroup_procs_write_finish(task, locked);
diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c
--- a/kernel/cgroup/cgroup.c
+++ b/kernel/cgroup/cgroup.c
@@ -61,6 +61,9 @@
#define CREATE_TRACE_POINTS
#include <trace/events/cgroup.h>
+#undef CREATE_TRACE_POINTS
+
+#include <trace/hooks/cgroup.h>
#define CGROUP_FILE_NAME_MAX (MAX_CGROUP_TYPE_NAMELEN + \
MAX_CFTYPE_NAME + 2)
@@ -2512,6 +2515,7 @@ static int cgroup_migrate_execute(struct cgroup_mgctx *mgctx)
do_each_subsys_mask(ss, ssid, mgctx->ss_mask) {
if (ss->attach) {
tset->ssid = ssid;
+ trace_android_vh_cgroup_attach(ss, tset);
ss->attach(tset);
}
} while_each_subsys_mask();
@@ -2812,11 +2816,13 @@ int cgroup_attach_task(struct cgroup *dst_cgrp, struct task_struct *leader,
}
struct task_struct *cgroup_procs_write_start(char *buf, bool threadgroup,
- bool *locked)
+ bool *locked,
+ struct cgroup *dst_cgrp)
__acquires(&cgroup_threadgroup_rwsem)
{
struct task_struct *tsk;
pid_t pid;
+ bool force_migration = false;
if (kstrtoint(strstrip(buf), 0, &pid) || pid < 0)
return ERR_PTR(-EINVAL);
@@ -2851,13 +2857,16 @@ struct task_struct *cgroup_procs_write_start(char *buf, bool threadgroup,
if (threadgroup)
tsk = tsk->group_leader;
+ if (tsk->flags & PF_KTHREAD)
+ trace_android_rvh_cgroup_force_kthread_migration(tsk, dst_cgrp, &force_migration);
+
/*
* kthreads may acquire PF_NO_SETAFFINITY during initialization.
* If userland migrates such a kthread to a non-root cgroup, it can
* become trapped in a cpuset, or RT kthread may be born in a
* cgroup with no rt_runtime allocated. Just say no.
*/
- if (tsk->no_cgroup_migration || (tsk->flags & PF_NO_SETAFFINITY)) {
+ if (!force_migration && (tsk->no_cgroup_migration || (tsk->flags & PF_NO_SETAFFINITY))) {
tsk = ERR_PTR(-EINVAL);
goto out_unlock_threadgroup;
}
@@ -4899,7 +4908,7 @@ static ssize_t __cgroup_procs_write(struct kernfs_open_file *of, char *buf,
if (!dst_cgrp)
return -ENODEV;
- task = cgroup_procs_write_start(buf, threadgroup, &locked);
+ task = cgroup_procs_write_start(buf, threadgroup, &locked, dst_cgrp);
ret = PTR_ERR_OR_ZERO(task);
if (ret)
goto out_unlock;
diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c
--- a/kernel/cgroup/cpuset.c
+++ b/kernel/cgroup/cpuset.c
@@ -66,6 +66,8 @@
#include <linux/cgroup.h>
#include <linux/wait.h>
+#include <trace/hooks/sched.h>
+
DEFINE_STATIC_KEY_FALSE(cpusets_pre_enable_key);
DEFINE_STATIC_KEY_FALSE(cpusets_enabled_key);
@@ -1090,6 +1092,18 @@ void rebuild_sched_domains(void)
cpus_read_unlock();
}
+static int update_cpus_allowed(struct cpuset *cs, struct task_struct *p,
+ const struct cpumask *new_mask)
+{
+ int ret = -EINVAL;
+
+ trace_android_rvh_update_cpus_allowed(p, cs->cpus_requested, new_mask, &ret);
+ if (!ret)
+ return ret;
+
+ return set_cpus_allowed_ptr(p, new_mask);
+}
+
/**
* update_tasks_cpumask - Update the cpumasks of tasks in the cpuset.
* @cs: the cpuset in which each task's cpus_allowed mask needs to be changed
@@ -1105,7 +1119,7 @@ static void update_tasks_cpumask(struct cpuset *cs)
css_task_iter_start(&cs->css, 0, &it);
while ((task = css_task_iter_next(&it)))
- set_cpus_allowed_ptr(task, cs->effective_cpus);
+ update_cpus_allowed(cs, task, cs->effective_cpus);
css_task_iter_end(&it);
}
@@ -2272,7 +2286,7 @@ static void cpuset_attach(struct cgroup_taskset *tset)
* can_attach beforehand should guarantee that this doesn't
* fail. TODO: have a better way to handle failure here
*/
- WARN_ON_ONCE(set_cpus_allowed_ptr(task, cpus_attach));
+ WARN_ON_ONCE(update_cpus_allowed(cs, task, cpus_attach));
cpuset_change_task_nodemask(task, &cpuset_attach_nodemask_to);
cpuset_update_task_spread_flag(cs, task);
diff --git a/kernel/cred.c b/kernel/cred.c
--- a/kernel/cred.c
+++ b/kernel/cred.c
@@ -17,6 +17,8 @@
#include <linux/cn_proc.h>
#include <linux/uidgid.h>
+#include <trace/hooks/creds.h>
+
#if 0
#define kdebug(FMT, ...) \
printk("[%-5.5s%5u] " FMT "\n", \
@@ -181,6 +183,7 @@ void exit_creds(struct task_struct *tsk)
key_put(tsk->cached_requested_key);
tsk->cached_requested_key = NULL;
#endif
+ trace_android_rvh_exit_creds(tsk, cred);
}
/**
@@ -499,6 +502,7 @@ int commit_creds(struct cred *new)
inc_rlimit_ucounts(new->ucounts, UCOUNT_RLIMIT_NPROC, 1);
rcu_assign_pointer(task->real_cred, new);
rcu_assign_pointer(task->cred, new);
+ trace_android_rvh_commit_creds(task, new);
if (new->user != old->user || new->user_ns != old->user_ns)
dec_rlimit_ucounts(old->ucounts, UCOUNT_RLIMIT_NPROC, 1);
alter_cred_subscribers(old, -2);
@@ -576,6 +580,7 @@ const struct cred *override_creds(const struct cred *new)
get_new_cred((struct cred *)new);
alter_cred_subscribers(new, 1);
rcu_assign_pointer(current->cred, new);
+ trace_android_rvh_override_creds(current, new);
alter_cred_subscribers(old, -1);
kdebug("override_creds() = %p{%d,%d}", old,
@@ -604,6 +609,7 @@ void revert_creds(const struct cred *old)
validate_creds(override);
alter_cred_subscribers(old, 1);
rcu_assign_pointer(current->cred, old);
+ trace_android_rvh_revert_creds(current, old);
alter_cred_subscribers(override, -1);
put_cred(override);
}
diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c
--- a/kernel/locking/rwsem.c
+++ b/kernel/locking/rwsem.c
@@ -30,7 +30,6 @@
#ifndef CONFIG_PREEMPT_RT
#include "lock_events.h"
-#include <trace/hooks/dtask.h>
/*
* The least significant 2 bits of the owner value has the following
@@ -1025,7 +1024,6 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat
wake_up_q(&wake_q);
/* wait to be given the lock */
- trace_android_vh_rwsem_read_wait_start(sem);
for (;;) {
set_current_state(state);
if (!smp_load_acquire(&waiter.task)) {
@@ -1045,7 +1043,6 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat
}
__set_current_state(TASK_RUNNING);
- trace_android_vh_rwsem_read_wait_finish(sem);
lockevent_inc(rwsem_rlock);
return sem;
@@ -1053,7 +1050,6 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat
rwsem_del_waiter(sem, &waiter);
raw_spin_unlock_irq(&sem->wait_lock);
__set_current_state(TASK_RUNNING);
- trace_android_vh_rwsem_read_wait_finish(sem);
lockevent_inc(rwsem_rlock_fail);
return ERR_PTR(-EINTR);
}
@@ -1121,7 +1117,6 @@ rwsem_down_write_slowpath(struct rw_semaphore *sem, int state)
wait:
/* wait until we successfully acquire the lock */
- trace_android_vh_rwsem_write_wait_start(sem);
set_current_state(state);
for (;;) {
if (rwsem_try_write_lock(sem, &waiter)) {
@@ -1160,14 +1155,12 @@ rwsem_down_write_slowpath(struct rw_semaphore *sem, int state)
raw_spin_lock_irq(&sem->wait_lock);
}
__set_current_state(TASK_RUNNING);
- trace_android_vh_rwsem_write_wait_finish(sem);
raw_spin_unlock_irq(&sem->wait_lock);
lockevent_inc(rwsem_wlock);
return sem;
out_nolock:
__set_current_state(TASK_RUNNING);
- trace_android_vh_rwsem_write_wait_finish(sem);
raw_spin_lock_irq(&sem->wait_lock);
rwsem_del_waiter(sem, &waiter);
if (!list_empty(&sem->wait_list))
diff --git a/kernel/module.c b/kernel/module.c
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -63,6 +63,10 @@
#define CREATE_TRACE_POINTS
#include <trace/events/module.h>
+#undef CREATE_TRACE_POINTS
+#include <trace/hooks/module.h>
+#include <trace/hooks/memory.h>
+
#ifndef ARCH_SHF_SMALL
#define ARCH_SHF_SMALL 0
#endif
@@ -2202,6 +2206,10 @@ static void free_module(struct module *mod)
/* This may be empty, but that's OK */
module_arch_freeing_init(mod);
+ trace_android_vh_set_memory_rw((unsigned long)mod->init_layout.base,
+ (mod->init_layout.size)>>PAGE_SHIFT);
+ trace_android_vh_set_memory_nx((unsigned long)mod->init_layout.base,
+ (mod->init_layout.size)>>PAGE_SHIFT);
module_memfree(mod->init_layout.base);
kfree(mod->args);
percpu_modfree(mod);
@@ -2210,6 +2218,10 @@ static void free_module(struct module *mod)
lockdep_free_key_range(mod->core_layout.base, mod->core_layout.size);
/* Finally, free the core (containing the module structure) */
+ trace_android_vh_set_memory_rw((unsigned long)mod->core_layout.base,
+ (mod->core_layout.size)>>PAGE_SHIFT);
+ trace_android_vh_set_memory_nx((unsigned long)mod->core_layout.base,
+ (mod->core_layout.size)>>PAGE_SHIFT);
module_memfree(mod->core_layout.base);
}
@@ -3626,7 +3638,15 @@ static void module_deallocate(struct module *mod, struct load_info *info)
{
percpu_modfree(mod);
module_arch_freeing_init(mod);
+ trace_android_vh_set_memory_rw((unsigned long)mod->init_layout.base,
+ (mod->init_layout.size)>>PAGE_SHIFT);
+ trace_android_vh_set_memory_nx((unsigned long)mod->init_layout.base,
+ (mod->init_layout.size)>>PAGE_SHIFT);
module_memfree(mod->init_layout.base);
+ trace_android_vh_set_memory_rw((unsigned long)mod->core_layout.base,
+ (mod->core_layout.size)>>PAGE_SHIFT);
+ trace_android_vh_set_memory_nx((unsigned long)mod->core_layout.base,
+ (mod->core_layout.size)>>PAGE_SHIFT);
module_memfree(mod->core_layout.base);
}
@@ -3784,8 +3804,13 @@ static noinline int do_init_module(struct module *mod)
rcu_assign_pointer(mod->kallsyms, &mod->core_kallsyms);
#endif
module_enable_ro(mod, true);
+ trace_android_vh_set_module_permit_after_init(mod);
mod_tree_remove_init(mod);
module_arch_freeing_init(mod);
+ trace_android_vh_set_memory_rw((unsigned long)mod->init_layout.base,
+ (mod->init_layout.size)>>PAGE_SHIFT);
+ trace_android_vh_set_memory_nx((unsigned long)mod->init_layout.base,
+ (mod->init_layout.size)>>PAGE_SHIFT);
mod->init_layout.base = NULL;
mod->init_layout.size = 0;
mod->init_layout.ro_size = 0;
@@ -3896,6 +3921,7 @@ static int complete_formation(struct module *mod, struct load_info *info)
module_enable_ro(mod, false);
module_enable_nx(mod);
module_enable_x(mod);
+ trace_android_vh_set_module_permit_before_init(mod);
/*
* Mark state as coming so strong_try_module_get() ignores us,
diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c
--- a/kernel/printk/printk.c
+++ b/kernel/printk/printk.c
@@ -54,6 +54,8 @@
#include <trace/events/initcall.h>
#define CREATE_TRACE_POINTS
#include <trace/events/printk.h>
+#undef CREATE_TRACE_POINTS
+#include <trace/hooks/printk.h>
#include "printk_ringbuffer.h"
#include "console_cmdline.h"
@@ -2501,6 +2503,12 @@ void resume_console(void)
*/
static int console_cpu_notify(unsigned int cpu)
{
+ int flag = 0;
+
+ trace_android_vh_printk_hotplug(&flag);
+ if (flag)
+ return 0;
+
if (!cpuhp_tasks_frozen) {
/* If trylock fails, someone else is doing the printing */
if (console_trylock())
diff --git a/kernel/sys.c b/kernel/sys.c
--- a/kernel/sys.c
+++ b/kernel/sys.c
@@ -76,6 +76,8 @@
#include "uid16.h"
+#include <trace/hooks/sys.h>
+
#ifndef SET_UNALIGN_CTL
# define SET_UNALIGN_CTL(a, b) (-EINVAL)
#endif
@@ -2686,6 +2688,7 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3,
error = -EINVAL;
break;
}
+ trace_android_vh_syscall_prctl_finished(option, me);
return error;
}
diff --git a/kernel/time/sched_clock.c b/kernel/time/sched_clock.c
--- a/kernel/time/sched_clock.c
+++ b/kernel/time/sched_clock.c
@@ -16,6 +16,7 @@
#include <linux/sched_clock.h>
#include <linux/seqlock.h>
#include <linux/bitops.h>
+#include <trace/hooks/epoch.h>
#include "timekeeping.h"
@@ -270,6 +271,7 @@ int sched_clock_suspend(void)
update_sched_clock();
hrtimer_cancel(&sched_clock_timer);
rd->read_sched_clock = suspended_sched_clock_read;
+ trace_android_vh_show_suspend_epoch_val(rd->epoch_ns, rd->epoch_cyc);
return 0;
}
@@ -281,6 +283,7 @@ void sched_clock_resume(void)
rd->epoch_cyc = cd.actual_read_sched_clock();
hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL_HARD);
rd->read_sched_clock = cd.actual_read_sched_clock;
+ trace_android_vh_show_resume_epoch_val(rd->epoch_cyc);
}
static struct syscore_ops sched_clock_ops = {
diff --git a/kernel/time/tick-common.c b/kernel/time/tick-common.c
--- a/kernel/time/tick-common.c
+++ b/kernel/time/tick-common.c
@@ -17,6 +17,7 @@
#include <linux/sched.h>
#include <linux/module.h>
#include <trace/events/power.h>
+#include <trace/hooks/sched.h>
#include <asm/irq_regs.h>
@@ -95,6 +96,7 @@ static void tick_periodic(int cpu)
write_seqcount_end(&jiffies_seq);
raw_spin_unlock(&jiffies_lock);
update_wall_time();
+ trace_android_vh_jiffies_update(NULL);
}
update_process_times(user_mode(get_irq_regs()));
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -26,6 +26,7 @@
#include <linux/posix-timers.h>
#include <linux/context_tracking.h>
#include <linux/mm.h>
+#include <trace/hooks/sched.h>
#include <asm/irq_regs.h>
@@ -193,8 +194,10 @@ static void tick_sched_do_timer(struct tick_sched *ts, ktime_t now)
#endif
/* Check, if the jiffies need an update */
- if (tick_do_timer_cpu == cpu)
+ if (tick_do_timer_cpu == cpu) {
tick_do_update_jiffies64(now);
+ trace_android_vh_jiffies_update(NULL);
+ }
if (ts->inidle)
ts->got_idle_tick = 1;
diff --git a/kernel/time/timer.c b/kernel/time/timer.c
--- a/kernel/time/timer.c
+++ b/kernel/time/timer.c
@@ -55,6 +55,8 @@
#define CREATE_TRACE_POINTS
#include <trace/events/timer.h>
+#undef CREATE_TRACE_POINTS
+#include <trace/hooks/timer.h>
__visible u64 jiffies_64 __cacheline_aligned_in_smp = INITIAL_JIFFIES;
@@ -503,6 +505,7 @@ static inline unsigned calc_index(unsigned long expires, unsigned lvl,
* Round up with level granularity to prevent this.
*/
expires = (expires + LVL_GRAN(lvl)) >> LVL_SHIFT(lvl);
+ trace_android_vh_timer_calc_index(lvl, &expires);
*bucket_expiry = expires << LVL_SHIFT(lvl);
return LVL_OFFS(lvl) + (expires & LVL_MASK);
}
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -49,6 +49,7 @@
#include <linux/fsnotify.h>
#include <linux/irq_work.h>
#include <linux/workqueue.h>
+#include <trace/hooks/ftrace_dump.h>
#include "trace.h"
#include "trace_output.h"
@@ -9713,8 +9714,17 @@ fs_initcall(tracer_init_tracefs);
static int trace_panic_handler(struct notifier_block *this,
unsigned long event, void *unused)
{
+ bool ftrace_check = false;
+
+ trace_android_vh_ftrace_oops_enter(&ftrace_check);
+
+ if (ftrace_check)
+ return NOTIFY_OK;
+
if (ftrace_dump_on_oops)
ftrace_dump(ftrace_dump_on_oops);
+
+ trace_android_vh_ftrace_oops_exit(&ftrace_check);
return NOTIFY_OK;
}
@@ -9728,6 +9738,13 @@ static int trace_die_handler(struct notifier_block *self,
unsigned long val,
void *data)
{
+ bool ftrace_check = false;
+
+ trace_android_vh_ftrace_oops_enter(&ftrace_check);
+
+ if (ftrace_check)
+ return NOTIFY_OK;
+
switch (val) {
case DIE_OOPS:
if (ftrace_dump_on_oops)
@@ -9736,6 +9753,8 @@ static int trace_die_handler(struct notifier_block *self,
default:
break;
}
+
+ trace_android_vh_ftrace_oops_exit(&ftrace_check);
return NOTIFY_OK;
}
@@ -9760,6 +9779,8 @@ static struct notifier_block trace_die_notifier = {
void
trace_printk_seq(struct trace_seq *s)
{
+ bool dump_printk = true;
+
/* Probably should print a warning here. */
if (s->seq.len >= TRACE_MAX_PRINT)
s->seq.len = TRACE_MAX_PRINT;
@@ -9775,7 +9796,9 @@ trace_printk_seq(struct trace_seq *s)
/* should be zero ended, but we are paranoid. */
s->buffer[s->seq.len] = 0;
- printk(KERN_TRACE "%s", s->buffer);
+ trace_android_vh_ftrace_dump_buffer(s, &dump_printk);
+ if (dump_printk)
+ printk(KERN_TRACE "%s", s->buffer);
trace_seq_init(s);
}
@@ -9808,6 +9831,8 @@ void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
unsigned int old_userobj;
unsigned long flags;
int cnt = 0, cpu;
+ bool ftrace_check = false;
+ unsigned long size;
/* Only allow one dump user at a time. */
if (atomic_inc_return(&dump_running) != 1) {
@@ -9837,6 +9862,8 @@ void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
for_each_tracing_cpu(cpu) {
atomic_inc(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled);
+ size = ring_buffer_size(iter.array_buffer->buffer, cpu);
+ trace_android_vh_ftrace_size_check(size, &ftrace_check);
}
old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ;
@@ -9844,6 +9871,9 @@ void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
/* don't look at user memory in panic mode */
tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
+ if (ftrace_check)
+ goto out_enable;
+
switch (oops_dump_mode) {
case DUMP_ALL:
iter.cpu_file = RING_BUFFER_ALL_CPUS;
@@ -9874,6 +9904,7 @@ void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
*/
while (!trace_empty(&iter)) {
+ ftrace_check = true;
if (!cnt)
printk(KERN_TRACE "---------------------------------\n");
@@ -9881,7 +9912,9 @@ void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
cnt++;
trace_iterator_reset(&iter);
- iter.iter_flags |= TRACE_FILE_LAT_FMT;
+ trace_android_vh_ftrace_format_check(&ftrace_check);
+ if (ftrace_check)
+ iter.iter_flags |= TRACE_FILE_LAT_FMT;
if (trace_find_next_entry_inc(&iter) != NULL) {
int ret;
diff --git a/kernel/trace/trace_preemptirq.c b/kernel/trace/trace_preemptirq.c
--- a/kernel/trace/trace_preemptirq.c
+++ b/kernel/trace/trace_preemptirq.c
@@ -14,6 +14,8 @@
#define CREATE_TRACE_POINTS
#include <trace/events/preemptirq.h>
+#undef CREATE_TRACE_POINTS
+#include <trace/hooks/preemptirq.h>
#ifdef CONFIG_TRACE_IRQFLAGS
/* Per-cpu variable to prevent redundant calls when IRQs already off */
@@ -28,8 +30,11 @@ static DEFINE_PER_CPU(int, tracing_irq_cpu);
void trace_hardirqs_on_prepare(void)
{
if (this_cpu_read(tracing_irq_cpu)) {
- if (!in_nmi())
+ if (!in_nmi()) {
trace_irq_enable(CALLER_ADDR0, CALLER_ADDR1);
+ trace_android_rvh_irqs_enable(CALLER_ADDR0,
+ CALLER_ADDR1);
+ }
tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1);
this_cpu_write(tracing_irq_cpu, 0);
}
@@ -40,8 +45,11 @@ NOKPROBE_SYMBOL(trace_hardirqs_on_prepare);
void trace_hardirqs_on(void)
{
if (this_cpu_read(tracing_irq_cpu)) {
- if (!in_nmi())
+ if (!in_nmi()) {
trace_irq_enable_rcuidle(CALLER_ADDR0, CALLER_ADDR1);
+ trace_android_rvh_irqs_enable(CALLER_ADDR0,
+ CALLER_ADDR1);
+ }
tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1);
this_cpu_write(tracing_irq_cpu, 0);
}
@@ -63,8 +71,11 @@ void trace_hardirqs_off_finish(void)
if (!this_cpu_read(tracing_irq_cpu)) {
this_cpu_write(tracing_irq_cpu, 1);
tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1);
- if (!in_nmi())
+ if (!in_nmi()) {
trace_irq_disable(CALLER_ADDR0, CALLER_ADDR1);
+ trace_android_rvh_irqs_disable(CALLER_ADDR0,
+ CALLER_ADDR1);
+ }
}
}
@@ -78,8 +89,11 @@ void trace_hardirqs_off(void)
if (!this_cpu_read(tracing_irq_cpu)) {
this_cpu_write(tracing_irq_cpu, 1);
tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1);
- if (!in_nmi())
+ if (!in_nmi()) {
trace_irq_disable_rcuidle(CALLER_ADDR0, CALLER_ADDR1);
+ trace_android_rvh_irqs_disable(CALLER_ADDR0,
+ CALLER_ADDR1);
+ }
}
}
EXPORT_SYMBOL(trace_hardirqs_off);
@@ -88,8 +102,11 @@ NOKPROBE_SYMBOL(trace_hardirqs_off);
__visible void trace_hardirqs_on_caller(unsigned long caller_addr)
{
if (this_cpu_read(tracing_irq_cpu)) {
- if (!in_nmi())
+ if (!in_nmi()) {
trace_irq_enable_rcuidle(CALLER_ADDR0, caller_addr);
+ trace_android_rvh_irqs_enable(CALLER_ADDR0,
+ caller_addr);
+ }
tracer_hardirqs_on(CALLER_ADDR0, caller_addr);
this_cpu_write(tracing_irq_cpu, 0);
}
@@ -107,8 +124,11 @@ __visible void trace_hardirqs_off_caller(unsigned long caller_addr)
if (!this_cpu_read(tracing_irq_cpu)) {
this_cpu_write(tracing_irq_cpu, 1);
tracer_hardirqs_off(CALLER_ADDR0, caller_addr);
- if (!in_nmi())
+ if (!in_nmi()) {
trace_irq_disable_rcuidle(CALLER_ADDR0, caller_addr);
+ trace_android_rvh_irqs_enable(CALLER_ADDR0,
+ caller_addr);
+ }
}
}
EXPORT_SYMBOL(trace_hardirqs_off_caller);
@@ -119,15 +139,19 @@ NOKPROBE_SYMBOL(trace_hardirqs_off_caller);
void trace_preempt_on(unsigned long a0, unsigned long a1)
{
- if (!in_nmi())
+ if (!in_nmi()) {
trace_preempt_enable_rcuidle(a0, a1);
+ trace_android_rvh_preempt_enable(a0, a1);
+ }
tracer_preempt_on(a0, a1);
}
void trace_preempt_off(unsigned long a0, unsigned long a1)
{
- if (!in_nmi())
+ if (!in_nmi()) {
trace_preempt_disable_rcuidle(a0, a1);
+ trace_android_rvh_preempt_disable(a0, a1);
+ }
tracer_preempt_off(a0, a1);
}
#endif
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -54,6 +54,10 @@
#include "workqueue_internal.h"
+#include <trace/hooks/wqlockup.h>
+/* events/workqueue.h uses default TRACE_INCLUDE_PATH */
+#undef TRACE_INCLUDE_PATH
+
enum {
/*
* worker_pool flags
@@ -5890,6 +5894,7 @@ static void wq_watchdog_timer_fn(struct timer_list *unused)
pr_cont_pool_info(pool);
pr_cont(" stuck for %us!\n",
jiffies_to_msecs(now - pool_ts) / 1000);
+ trace_android_vh_wq_lockup_pool(pool->cpu, pool_ts);
}
}