From ca95fb34a35386c9385a7fb4ccec8dd51eadc94b Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Date: Tue, 16 Nov 2021 15:08:27 +0100 Subject: [PATCH] Add kernel support for context switch events These events only show if a thread has been removed from the CPU, but doesn't provide information about what other thread caused the context switch. This type of information can be obtained even with strict /proc/sys/kernel/perf_event_paranoid settings. --- CMakeLists.txt | 1 + cfg/cpu-kernel-context-switch.cfg | 43 +++++++++++ cfg/thread-kernel-context-switch.cfg | 43 +++++++++++ doc/emu_events.txt | 4 ++ emu.c | 2 + emu.h | 45 +++++++----- emu_kernel.c | 102 +++++++++++++++++++++++++++ pcf.c | 23 ++++++ 8 files changed, 244 insertions(+), 19 deletions(-) create mode 100644 cfg/cpu-kernel-context-switch.cfg create mode 100644 cfg/thread-kernel-context-switch.cfg create mode 100644 emu_kernel.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 946486d..ca7b71f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -80,6 +80,7 @@ add_executable(ovniemu emu_ovni.c emu_tampi.c emu_nanos6.c + emu_kernel.c trace.c ovni.c parson.c diff --git a/cfg/cpu-kernel-context-switch.cfg b/cfg/cpu-kernel-context-switch.cfg new file mode 100644 index 0000000..caedc9a --- /dev/null +++ b/cfg/cpu-kernel-context-switch.cfg @@ -0,0 +1,43 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 1 + + +################################################################################ +< NEW DISPLAYING WINDOW CPU: Context switches of the ACTIVE thread > +################################################################################ +window_name CPU: Context switches of the ACTIVE thread +window_type single +window_id 1 +window_position_x 960 +window_position_y 287 +window_width 954 +window_height 236 +window_comm_lines_enabled true +window_flags_enabled true +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 5.000000000000 +window_minimum_y 1.000000000000 +window_compute_y_max true +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open true +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_filter_module evt_type 1 105 +window_filter_module evt_type_label 1 "CPU: Context switches of the ACTIVE thread" +window_synchronize 1 + diff --git a/cfg/thread-kernel-context-switch.cfg b/cfg/thread-kernel-context-switch.cfg new file mode 100644 index 0000000..1705146 --- /dev/null +++ b/cfg/thread-kernel-context-switch.cfg @@ -0,0 +1,43 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 1 + + +################################################################################ +< NEW DISPLAYING WINDOW Thread: Context switches of the CURRENT thread > +################################################################################ +window_name Thread: Context switches of the CURRENT thread +window_type single +window_id 1 +window_position_x 960 +window_position_y 287 +window_width 954 +window_height 236 +window_comm_lines_enabled true +window_flags_enabled true +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 5.000000000000 +window_minimum_y 1.000000000000 +window_compute_y_max true +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open true +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_filter_module evt_type 1 55 +window_filter_module evt_type_label 1 "Thread: Context switches of the CURRENT thread" +window_synchronize 1 + diff --git a/doc/emu_events.txt b/doc/emu_events.txt index 4986595..076d48d 100644 --- a/doc/emu_events.txt +++ b/doc/emu_events.txt @@ -127,3 +127,7 @@ LS] Ends the submit of a task LP[ Begins the spawn of a function LP] Ends the spawn of a function +-------------------- Kernel (model=K) ---------------------- + +KCO Is out of the CPU due to a context switch +KCI Is back in the CPU due to a context switch diff --git a/emu.c b/emu.c index 8bed5b0..eb8c31b 100644 --- a/emu.c +++ b/emu.c @@ -266,6 +266,7 @@ hook_init(struct ovni_emu *emu) hook_init_tampi(emu); hook_init_openmp(emu); hook_init_nanos6(emu); + hook_init_kernel(emu); } static void @@ -278,6 +279,7 @@ hook_pre(struct ovni_emu *emu) case 'T': hook_pre_tampi(emu); break; case 'M': hook_pre_openmp(emu); break; case 'L': hook_pre_nanos6(emu); break; + case 'K': hook_pre_kernel(emu); break; default: break; } diff --git a/emu.h b/emu.h index 17c0c96..087bfb9 100644 --- a/emu.h +++ b/emu.h @@ -99,6 +99,10 @@ enum nanos6_state { ST_NANOS6_SPAWN = 8, }; +enum kernel_cs_state { + ST_KERNEL_CSOUT = 3, +}; + struct ovni_ethread; struct ovni_eproc; @@ -149,6 +153,8 @@ enum chan { CHAN_OPENMP_MODE, CHAN_NANOS6_SUBSYSTEM, + CHAN_KERNEL_CS, + CHAN_MAX }; @@ -170,25 +176,23 @@ enum chan_dirty { /* Same order as `enum chan` */ static const int chan_to_prvtype[CHAN_MAX][3] = { - /* Channel TH CPU */ - { CHAN_OVNI_PID, 10, 60 }, - { CHAN_OVNI_TID, 11, 61 }, - { CHAN_OVNI_NRTHREADS, -1, 62 }, - { CHAN_OVNI_STATE, 13, -1 }, - { CHAN_OVNI_APPID, 14, 64 }, /* Not used */ - { CHAN_OVNI_CPU, 15, -1 }, - { CHAN_OVNI_FLUSH, 16, 66 }, - - { CHAN_NOSV_TASKID, 20, 70 }, - { CHAN_NOSV_TYPEID, 21, 71 }, - { CHAN_NOSV_APPID, 22, 72 }, - { CHAN_NOSV_SUBSYSTEM, 23, 73 }, - - { CHAN_TAMPI_MODE, 30, 80 }, - - { CHAN_OPENMP_MODE, 40, 90 }, - - { CHAN_NANOS6_SUBSYSTEM, 50, 100 }, + /* FIXME: Use odd/even identifiers for thread and cpu */ + /* Channel TH CPU */ + { CHAN_OVNI_PID, 10, 60 }, + { CHAN_OVNI_TID, 11, 61 }, + { CHAN_OVNI_NRTHREADS, -1, 62 }, + { CHAN_OVNI_STATE, 13, -1 }, + { CHAN_OVNI_APPID, 14, 64 }, /* Not used */ + { CHAN_OVNI_CPU, 15, -1 }, + { CHAN_OVNI_FLUSH, 16, 66 }, + { CHAN_NOSV_TASKID, 20, 70 }, + { CHAN_NOSV_TYPEID, 21, 71 }, + { CHAN_NOSV_APPID, 22, 72 }, + { CHAN_NOSV_SUBSYSTEM, 23, 73 }, + { CHAN_TAMPI_MODE, 30, 80 }, + { CHAN_OPENMP_MODE, 40, 90 }, + { CHAN_NANOS6_SUBSYSTEM, 50, 100 }, + { CHAN_KERNEL_CS, 55, 105 }, }; #define CHAN_PRV_TH(id) chan_to_prvtype[id][CHAN_TH] @@ -493,6 +497,9 @@ void hook_pre_openmp(struct ovni_emu *emu); void hook_init_nanos6(struct ovni_emu *emu); void hook_pre_nanos6(struct ovni_emu *emu); +void hook_init_kernel(struct ovni_emu *emu); +void hook_pre_kernel(struct ovni_emu *emu); + struct ovni_cpu *emu_get_cpu(struct ovni_loom *loom, int cpuid); struct ovni_ethread *emu_get_thread(struct ovni_eproc *proc, int tid); diff --git a/emu_kernel.c b/emu_kernel.c new file mode 100644 index 0000000..f6f82a2 --- /dev/null +++ b/emu_kernel.c @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2021 Barcelona Supercomputing Center (BSC) + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include +#include "uthash.h" + +#include "ovni.h" +#include "trace.h" +#include "emu.h" +#include "prv.h" +#include "chan.h" + +/* --------------------------- init ------------------------------- */ + +void +hook_init_kernel(struct ovni_emu *emu) +{ + struct ovni_ethread *th; + struct ovni_cpu *cpu; + size_t i; + int row; + FILE *prv_th, *prv_cpu; + int64_t *clock; + struct ovni_chan **uth, **ucpu; + + clock = &emu->delta_time; + prv_th = emu->prv_thread; + prv_cpu = emu->prv_cpu; + + /* Init the channels in all threads */ + for(i=0; itotal_nthreads; i++) + { + th = emu->global_thread[i]; + row = th->gindex + 1; + uth = &emu->th_chan; + + chan_th_init(th, uth, CHAN_KERNEL_CS, CHAN_TRACK_NONE, 0, 1, 1, row, prv_th, clock); + } + + /* Init the channels in all cpus */ + for(i=0; itotal_ncpus; i++) + { + cpu = emu->global_cpu[i]; + row = cpu->gindex + 1; + ucpu = &emu->cpu_chan; + + chan_cpu_init(cpu, ucpu, CHAN_KERNEL_CS, CHAN_TRACK_TH_ACTIVE, 0, 0, 1, row, prv_cpu, clock); + } +} + +/* --------------------------- pre ------------------------------- */ + +static void +context_switch(struct ovni_emu *emu) +{ + struct ovni_ethread *th; + struct ovni_chan *chan; + + th = emu->cur_thread; + chan = &th->chan[CHAN_KERNEL_CS]; + + switch(emu->cur_ev->header.value) + { + case 'O': + chan_push(chan, ST_KERNEL_CSOUT); + break; + case 'I': + chan_pop(chan, ST_KERNEL_CSOUT); + break; + default: + err("unexpected value '%c' (expecting 'O' or 'I')\n", + emu->cur_ev->header.value); + abort(); + } +} + +void +hook_pre_kernel(struct ovni_emu *emu) +{ + assert(emu->cur_ev->header.model == 'K'); + + switch(emu->cur_ev->header.category) + { + case 'C': context_switch(emu); break; + default: + break; + } +} diff --git a/pcf.c b/pcf.c index 86fbd4d..b2e673a 100644 --- a/pcf.c +++ b/pcf.c @@ -383,6 +383,27 @@ static const struct event_type thread_nanos6_mode = { nanos6_mode_values }; +/* ---------------- CHAN_KERNEL_CS ---------------- */ + +struct event_value kernel_cs_values[] = { + { ST_NULL, "NULL" }, + { ST_TOO_MANY_TH, "Kernel CS: Unknown, multiple threads running" }, + { ST_KERNEL_CSOUT, "Context switch: Out of the CPU" }, + { -1, NULL }, +}; + +struct event_type cpu_kernel_cs = { + 0, CHAN_KERNEL_CS, CHAN_CPU, + "CPU: Context switches of the ACTIVE thread", + kernel_cs_values +}; + +struct event_type thread_kernel_cs = { + 0, CHAN_KERNEL_CS, CHAN_TH, + "Thread: Context switches of the CURRENT thread", + kernel_cs_values +}; + /* ----------------------------------------------- */ static void @@ -475,6 +496,7 @@ write_events(FILE *f, struct ovni_emu *emu) write_event_type(f, &thread_tampi_mode); write_event_type(f, &thread_openmp_mode); write_event_type(f, &thread_nanos6_mode); + write_event_type(f, &thread_kernel_cs); /* CPU */ write_event_type(f, &cpu_ovni_pid); @@ -491,6 +513,7 @@ write_events(FILE *f, struct ovni_emu *emu) write_event_type(f, &cpu_tampi_mode); write_event_type(f, &cpu_openmp_mode); write_event_type(f, &cpu_nanos6_mode); + write_event_type(f, &cpu_kernel_cs); /* Custom */ write_cpu_type(f, &thread_cpu_affinity, emu);