AcoSail/kernel_optimize_test
8
0
Fork 0
forked from luck/tmp_suning_uos_patched
Code Pull Requests Activity

perf: Reduce stack usage of perf_output_begin()

Browse Source 
__perf_output_begin() has an on-stack struct perf_sample_data in the
unlikely case it needs to generate a LOST record. However, every call
to perf_output_begin() must already have a perf_sample_data on-stack.

Reported-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lkml.kernel.org/r/20201030151954.985416146@infradead.org
This commit is contained in:
Peter Zijlstra 2020-10-30 15:50:32 +01:00
parent 7bdb157cde
commit 267fb27352
6 changed files with 37 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
arch/powerpc/perf/imc-pmu.c
View File

@ -1336,7 +1336,7 @@ static void dump_trace_imc_data(struct perf_event *event)
/* If this is a valid record, create the sample */ /* If this is a valid record, create the sample */
struct perf_output_handle handle; struct perf_output_handle handle;
if (perf_output_begin(&handle, event, header.size)) if (perf_output_begin(&handle, &data, event, header.size))
return; return;
perf_output_sample(&handle, &header, &data, event); perf_output_sample(&handle, &header, &data, event);

2
arch/s390/kernel/perf_cpum_sf.c
View File

@ -672,7 +672,7 @@ static void cpumsf_output_event_pid(struct perf_event *event,
rcu_read_lock(); rcu_read_lock();
perf_prepare_sample(&header, data, event, regs); perf_prepare_sample(&header, data, event, regs);
if (perf_output_begin(&handle, event, header.size)) if (perf_output_begin(&handle, data, event, header.size))
goto out; goto out;
/* Update the process ID (see also kernel/events/core.c) */ /* Update the process ID (see also kernel/events/core.c) */

4
arch/x86/events/intel/ds.c
View File

@ -642,8 +642,8 @@ int intel_pmu_drain_bts_buffer(void)
rcu_read_lock(); rcu_read_lock();
perf_prepare_sample(&header, &data, event, &regs); perf_prepare_sample(&header, &data, event, &regs);
if (perf_output_begin(&handle, event, header.size * if (perf_output_begin(&handle, &data, event,
(top - base - skip))) header.size * (top - base - skip)))
goto unlock; goto unlock;
for (at = base; at < top; at++) { for (at = base; at < top; at++) {

7
include/linux/perf_event.h
View File

@ -1400,11 +1400,14 @@ perf_event_addr_filters(struct perf_event *event)
extern void perf_event_addr_filters_sync(struct perf_event *event); extern void perf_event_addr_filters_sync(struct perf_event *event);
extern int perf_output_begin(struct perf_output_handle *handle, extern int perf_output_begin(struct perf_output_handle *handle,
struct perf_sample_data *data,
struct perf_event *event, unsigned int size); struct perf_event *event, unsigned int size);
extern int perf_output_begin_forward(struct perf_output_handle *handle, extern int perf_output_begin_forward(struct perf_output_handle *handle,
struct perf_event *event, struct perf_sample_data *data,
unsigned int size); struct perf_event *event,
unsigned int size);
extern int perf_output_begin_backward(struct perf_output_handle *handle, extern int perf_output_begin_backward(struct perf_output_handle *handle,
struct perf_sample_data *data,
struct perf_event *event, struct perf_event *event,
unsigned int size); unsigned int size);

32
kernel/events/core.c
View File

@ -7186,6 +7186,7 @@ __perf_event_output(struct perf_event *event,
struct perf_sample_data *data, struct perf_sample_data *data,
struct pt_regs *regs, struct pt_regs *regs,
int (*output_begin)(struct perf_output_handle *, int (*output_begin)(struct perf_output_handle *,
struct perf_sample_data *,
struct perf_event *, struct perf_event *,
unsigned int)) unsigned int))
{ {
@ -7198,7 +7199,7 @@ __perf_event_output(struct perf_event *event,
perf_prepare_sample(&header, data, event, regs); perf_prepare_sample(&header, data, event, regs);
err = output_begin(&handle, event, header.size); err = output_begin(&handle, data, event, header.size);
if (err) if (err)
goto exit; goto exit;
@ -7264,7 +7265,7 @@ perf_event_read_event(struct perf_event *event,
int ret; int ret;
perf_event_header__init_id(&read_event.header, &sample, event); perf_event_header__init_id(&read_event.header, &sample, event);
ret = perf_output_begin(&handle, event, read_event.header.size); ret = perf_output_begin(&handle, &sample, event, read_event.header.size);
if (ret) if (ret)
return; return;
@ -7533,7 +7534,7 @@ static void perf_event_task_output(struct perf_event *event,
perf_event_header__init_id(&task_event->event_id.header, &sample, event); perf_event_header__init_id(&task_event->event_id.header, &sample, event);
ret = perf_output_begin(&handle, event, ret = perf_output_begin(&handle, &sample, event,
task_event->event_id.header.size); task_event->event_id.header.size);
if (ret) if (ret)
goto out; goto out;
@ -7636,7 +7637,7 @@ static void perf_event_comm_output(struct perf_event *event,
return; return;
perf_event_header__init_id(&comm_event->event_id.header, &sample, event); perf_event_header__init_id(&comm_event->event_id.header, &sample, event);
ret = perf_output_begin(&handle, event, ret = perf_output_begin(&handle, &sample, event,
comm_event->event_id.header.size); comm_event->event_id.header.size);
if (ret) if (ret)
@ -7736,7 +7737,7 @@ static void perf_event_namespaces_output(struct perf_event *event,
perf_event_header__init_id(&namespaces_event->event_id.header, perf_event_header__init_id(&namespaces_event->event_id.header,
&sample, event); &sample, event);
ret = perf_output_begin(&handle, event, ret = perf_output_begin(&handle, &sample, event,
namespaces_event->event_id.header.size); namespaces_event->event_id.header.size);
if (ret) if (ret)
goto out; goto out;
@ -7863,7 +7864,7 @@ static void perf_event_cgroup_output(struct perf_event *event, void *data)
perf_event_header__init_id(&cgroup_event->event_id.header, perf_event_header__init_id(&cgroup_event->event_id.header,
&sample, event); &sample, event);
ret = perf_output_begin(&handle, event, ret = perf_output_begin(&handle, &sample, event,
cgroup_event->event_id.header.size); cgroup_event->event_id.header.size);
if (ret) if (ret)
goto out; goto out;
@ -7989,7 +7990,7 @@ static void perf_event_mmap_output(struct perf_event *event,
} }
perf_event_header__init_id(&mmap_event->event_id.header, &sample, event); perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);
ret = perf_output_begin(&handle, event, ret = perf_output_begin(&handle, &sample, event,
mmap_event->event_id.header.size); mmap_event->event_id.header.size);
if (ret) if (ret)
goto out; goto out;
@ -8299,7 +8300,7 @@ void perf_event_aux_event(struct perf_event *event, unsigned long head,
int ret; int ret;
perf_event_header__init_id(&rec.header, &sample, event); perf_event_header__init_id(&rec.header, &sample, event);
ret = perf_output_begin(&handle, event, rec.header.size); ret = perf_output_begin(&handle, &sample, event, rec.header.size);
if (ret) if (ret)
return; return;
@ -8333,7 +8334,7 @@ void perf_log_lost_samples(struct perf_event *event, u64 lost)
perf_event_header__init_id(&lost_samples_event.header, &sample, event); perf_event_header__init_id(&lost_samples_event.header, &sample, event);
ret = perf_output_begin(&handle, event, ret = perf_output_begin(&handle, &sample, event,
lost_samples_event.header.size); lost_samples_event.header.size);
if (ret) if (ret)
return; return;
@ -8388,7 +8389,7 @@ static void perf_event_switch_output(struct perf_event *event, void *data)
perf_event_header__init_id(&se->event_id.header, &sample, event); perf_event_header__init_id(&se->event_id.header, &sample, event);
ret = perf_output_begin(&handle, event, se->event_id.header.size); ret = perf_output_begin(&handle, &sample, event, se->event_id.header.size);
if (ret) if (ret)
return; return;
@ -8463,7 +8464,7 @@ static void perf_log_throttle(struct perf_event *event, int enable)
perf_event_header__init_id(&throttle_event.header, &sample, event); perf_event_header__init_id(&throttle_event.header, &sample, event);
ret = perf_output_begin(&handle, event, ret = perf_output_begin(&handle, &sample, event,
throttle_event.header.size); throttle_event.header.size);
if (ret) if (ret)
return; return;
@ -8506,7 +8507,7 @@ static void perf_event_ksymbol_output(struct perf_event *event, void *data)
perf_event_header__init_id(&ksymbol_event->event_id.header, perf_event_header__init_id(&ksymbol_event->event_id.header,
&sample, event); &sample, event);
ret = perf_output_begin(&handle, event, ret = perf_output_begin(&handle, &sample, event,
ksymbol_event->event_id.header.size); ksymbol_event->event_id.header.size);
if (ret) if (ret)
return; return;
@ -8596,7 +8597,7 @@ static void perf_event_bpf_output(struct perf_event *event, void *data)
perf_event_header__init_id(&bpf_event->event_id.header, perf_event_header__init_id(&bpf_event->event_id.header,
&sample, event); &sample, event);
ret = perf_output_begin(&handle, event, ret = perf_output_begin(&handle, data, event,
bpf_event->event_id.header.size); bpf_event->event_id.header.size);
if (ret) if (ret)
return; return;
@ -8705,7 +8706,8 @@ static void perf_event_text_poke_output(struct perf_event *event, void *data)
perf_event_header__init_id(&text_poke_event->event_id.header, &sample, event); perf_event_header__init_id(&text_poke_event->event_id.header, &sample, event);
ret = perf_output_begin(&handle, event, text_poke_event->event_id.header.size); ret = perf_output_begin(&handle, &sample, event,
text_poke_event->event_id.header.size);
if (ret) if (ret)
return; return;
@ -8786,7 +8788,7 @@ static void perf_log_itrace_start(struct perf_event *event)
rec.tid = perf_event_tid(event, current); rec.tid = perf_event_tid(event, current);
perf_event_header__init_id(&rec.header, &sample, event); perf_event_header__init_id(&rec.header, &sample, event);
ret = perf_output_begin(&handle, event, rec.header.size); ret = perf_output_begin(&handle, &sample, event, rec.header.size);
if (ret) if (ret)
return; return;

20
kernel/events/ring_buffer.c
View File

@ -147,6 +147,7 @@ ring_buffer_has_space(unsigned long head, unsigned long tail,
static __always_inline int static __always_inline int
__perf_output_begin(struct perf_output_handle *handle, __perf_output_begin(struct perf_output_handle *handle,
struct perf_sample_data *data,
struct perf_event *event, unsigned int size, struct perf_event *event, unsigned int size,
bool backward) bool backward)
{ {
@ -237,18 +238,16 @@ __perf_output_begin(struct perf_output_handle *handle,
handle->size = (1UL << page_shift) - offset; handle->size = (1UL << page_shift) - offset;
if (unlikely(have_lost)) { if (unlikely(have_lost)) {
struct perf_sample_data sample_data;
lost_event.header.size = sizeof(lost_event); lost_event.header.size = sizeof(lost_event);
lost_event.header.type = PERF_RECORD_LOST; lost_event.header.type = PERF_RECORD_LOST;
lost_event.header.misc = 0; lost_event.header.misc = 0;
lost_event.id = event->id; lost_event.id = event->id;
lost_event.lost = local_xchg(&rb->lost, 0); lost_event.lost = local_xchg(&rb->lost, 0);
perf_event_header__init_id(&lost_event.header, /* XXX mostly redundant; @data is already fully initializes */
&sample_data, event); perf_event_header__init_id(&lost_event.header, data, event);
perf_output_put(handle, lost_event); perf_output_put(handle, lost_event);
perf_event__output_id_sample(event, handle, &sample_data); perf_event__output_id_sample(event, handle, data);
} }
return 0; return 0;
@ -263,22 +262,25 @@ __perf_output_begin(struct perf_output_handle *handle,
} }
int perf_output_begin_forward(struct perf_output_handle *handle, int perf_output_begin_forward(struct perf_output_handle *handle,
struct perf_event *event, unsigned int size) struct perf_sample_data *data,
struct perf_event *event, unsigned int size)
{ {
return __perf_output_begin(handle, event, size, false); return __perf_output_begin(handle, data, event, size, false);
} }
int perf_output_begin_backward(struct perf_output_handle *handle, int perf_output_begin_backward(struct perf_output_handle *handle,
struct perf_sample_data *data,
struct perf_event *event, unsigned int size) struct perf_event *event, unsigned int size)
{ {
return __perf_output_begin(handle, event, size, true); return __perf_output_begin(handle, data, event, size, true);
} }
int perf_output_begin(struct perf_output_handle *handle, int perf_output_begin(struct perf_output_handle *handle,
struct perf_sample_data *data,
struct perf_event *event, unsigned int size) struct perf_event *event, unsigned int size)
{ {
return __perf_output_begin(handle, event, size, return __perf_output_begin(handle, data, event, size,
unlikely(is_write_backward(event))); unlikely(is_write_backward(event)));
} }