Add kernel support for context switch events

These events only show if a thread has been removed from the CPU, but
doesn't provide information about what other thread caused the context
switch. This type of information can be obtained even with strict
/proc/sys/kernel/perf_event_paranoid settings.

CMakeLists.txt

@@ -80,6 +80,7 @@ add_executable(ovniemu
   emu_ovni.c
   emu_tampi.c
   emu_nanos6.c
+  emu_kernel.c
   trace.c
   ovni.c
   parson.c

cfg/cpu-kernel-context-switch.cfg

@@ -0,0 +1,43 @@
+#ParaverCFG
+ConfigFile.Version: 3.4
+ConfigFile.NumWindows: 1
+
+
+################################################################################
+< NEW DISPLAYING WINDOW CPU: Context switches of the ACTIVE thread >
+################################################################################
+window_name CPU: Context switches of the ACTIVE thread
+window_type single
+window_id 1
+window_position_x 960
+window_position_y 287
+window_width 954
+window_height 236
+window_comm_lines_enabled true
+window_flags_enabled true
+window_noncolor_mode true
+window_logical_filtered true
+window_physical_filtered false
+window_comm_fromto true
+window_comm_tagsize true
+window_comm_typeval true
+window_units Microseconds
+window_maximum_y 5.000000000000
+window_minimum_y 1.000000000000
+window_compute_y_max true
+window_level thread
+window_scale_relative 1.000000000000
+window_end_time_relative 1.000000000000
+window_object appl { 1, { All } }
+window_begin_time_relative 0.000000000000
+window_open true
+window_drawmode draw_maximum
+window_drawmode_rows draw_maximum
+window_pixel_size 1
+window_labels_to_draw 1
+window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } }
+window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } }
+window_filter_module evt_type 1 105
+window_filter_module evt_type_label 1 "CPU: Context switches of the ACTIVE thread"
+window_synchronize 1
+

cfg/thread-kernel-context-switch.cfg

@@ -0,0 +1,43 @@
+#ParaverCFG
+ConfigFile.Version: 3.4
+ConfigFile.NumWindows: 1
+
+
+################################################################################
+< NEW DISPLAYING WINDOW Thread: Context switches of the CURRENT thread >
+################################################################################
+window_name Thread: Context switches of the CURRENT thread
+window_type single
+window_id 1
+window_position_x 960
+window_position_y 287
+window_width 954
+window_height 236
+window_comm_lines_enabled true
+window_flags_enabled true
+window_noncolor_mode true
+window_logical_filtered true
+window_physical_filtered false
+window_comm_fromto true
+window_comm_tagsize true
+window_comm_typeval true
+window_units Microseconds
+window_maximum_y 5.000000000000
+window_minimum_y 1.000000000000
+window_compute_y_max true
+window_level thread
+window_scale_relative 1.000000000000
+window_end_time_relative 1.000000000000
+window_object appl { 1, { All } }
+window_begin_time_relative 0.000000000000
+window_open true
+window_drawmode draw_maximum
+window_drawmode_rows draw_maximum
+window_pixel_size 1
+window_labels_to_draw 1
+window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } }
+window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } }
+window_filter_module evt_type 1 55
+window_filter_module evt_type_label 1 "Thread: Context switches of the CURRENT thread"
+window_synchronize 1
+

doc/emu_events.txt

@@ -127,3 +127,7 @@ LS]	Ends the submit of a task
 LP[	Begins the spawn of a function
 LP]	Ends the spawn of a function
 
+-------------------- Kernel (model=K) ----------------------
+
+KCO	Is out of the CPU due to a context switch
+KCI	Is back in the CPU due to a context switch

emu.c

@@ -266,6 +266,7 @@ hook_init(struct ovni_emu *emu)
 	hook_init_tampi(emu);
 	hook_init_openmp(emu);
 	hook_init_nanos6(emu);
+	hook_init_kernel(emu);
 }
 
 static void
@@ -278,6 +279,7 @@ hook_pre(struct ovni_emu *emu)
 		case 'T': hook_pre_tampi(emu); break;
 		case 'M': hook_pre_openmp(emu); break;
 		case 'L': hook_pre_nanos6(emu); break;
+		case 'K': hook_pre_kernel(emu); break;
 		default:
 			break;
 	}

emu.h

@@ -99,6 +99,10 @@ enum nanos6_state {
 	ST_NANOS6_SPAWN = 8,
 };
 
+enum kernel_cs_state {
+	ST_KERNEL_CSOUT = 3,
+};
+
 struct ovni_ethread;
 struct ovni_eproc;
 
@@ -149,6 +153,8 @@ enum chan {
 	CHAN_OPENMP_MODE,
 	CHAN_NANOS6_SUBSYSTEM,
 
+	CHAN_KERNEL_CS,
+
 	CHAN_MAX
 };
 
@@ -170,25 +176,23 @@ enum chan_dirty {
 
 /* Same order as `enum chan` */
 static const int chan_to_prvtype[CHAN_MAX][3] = {
-	/* Channel		TH  CPU */
+	/* FIXME: Use odd/even identifiers for thread and cpu */
-	{ CHAN_OVNI_PID,	10, 60 },
+	/* Channel                TH   CPU */
-	{ CHAN_OVNI_TID,	11, 61 },
+	{ CHAN_OVNI_PID,          10,  60  },
-	{ CHAN_OVNI_NRTHREADS,	-1, 62 },
+	{ CHAN_OVNI_TID,          11,  61  },
-	{ CHAN_OVNI_STATE,	13, -1 },
+	{ CHAN_OVNI_NRTHREADS,    -1,  62  },
-	{ CHAN_OVNI_APPID,	14, 64 }, /* Not used */
+	{ CHAN_OVNI_STATE,        13,  -1  },
-	{ CHAN_OVNI_CPU,	15, -1 },
+	{ CHAN_OVNI_APPID,        14,  64  }, /* Not used */
-	{ CHAN_OVNI_FLUSH,	16, 66 },
+	{ CHAN_OVNI_CPU,          15,  -1  },
-
+	{ CHAN_OVNI_FLUSH,        16,  66  },
-	{ CHAN_NOSV_TASKID,	20, 70 },
+	{ CHAN_NOSV_TASKID,       20,  70  },
-	{ CHAN_NOSV_TYPEID,	21, 71 },
+	{ CHAN_NOSV_TYPEID,       21,  71  },
-	{ CHAN_NOSV_APPID,	22, 72 },
+	{ CHAN_NOSV_APPID,        22,  72  },
-	{ CHAN_NOSV_SUBSYSTEM,	23, 73 },
+	{ CHAN_NOSV_SUBSYSTEM,    23,  73  },
-
+	{ CHAN_TAMPI_MODE,        30,  80  },
-	{ CHAN_TAMPI_MODE,	30, 80 },
+	{ CHAN_OPENMP_MODE,       40,  90  },
-
+	{ CHAN_NANOS6_SUBSYSTEM,  50, 100  },
-	{ CHAN_OPENMP_MODE,	40, 90 },
+	{ CHAN_KERNEL_CS,         55, 105  },
-
-	{ CHAN_NANOS6_SUBSYSTEM,	50, 100 },
 };
 
 #define CHAN_PRV_TH(id) chan_to_prvtype[id][CHAN_TH]
@@ -493,6 +497,9 @@ void hook_pre_openmp(struct ovni_emu *emu);
 void hook_init_nanos6(struct ovni_emu *emu);
 void hook_pre_nanos6(struct ovni_emu *emu);
 
+void hook_init_kernel(struct ovni_emu *emu);
+void hook_pre_kernel(struct ovni_emu *emu);
+
 struct ovni_cpu *emu_get_cpu(struct ovni_loom *loom, int cpuid);
 
 struct ovni_ethread *emu_get_thread(struct ovni_eproc *proc, int tid);

emu_kernel.c

@@ -0,0 +1,102 @@
+/*
+ * Copyright (c) 2021 Barcelona Supercomputing Center (BSC)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include <assert.h>
+#include "uthash.h"
+
+#include "ovni.h"
+#include "trace.h"
+#include "emu.h"
+#include "prv.h"
+#include "chan.h"
+
+/* --------------------------- init ------------------------------- */
+
+void
+hook_init_kernel(struct ovni_emu *emu)
+{
+	struct ovni_ethread *th;
+	struct ovni_cpu *cpu;
+	size_t i;
+	int row;
+	FILE *prv_th, *prv_cpu;
+	int64_t *clock;
+	struct ovni_chan **uth, **ucpu;
+
+	clock = &emu->delta_time;
+	prv_th = emu->prv_thread;
+	prv_cpu = emu->prv_cpu;
+
+	/* Init the channels in all threads */
+	for(i=0; i<emu->total_nthreads; i++)
+	{
+		th = emu->global_thread[i];
+		row = th->gindex + 1;
+		uth = &emu->th_chan;
+
+		chan_th_init(th, uth, CHAN_KERNEL_CS, CHAN_TRACK_NONE, 0, 1, 1, row, prv_th, clock);
+	}
+
+	/* Init the channels in all cpus */
+	for(i=0; i<emu->total_ncpus; i++)
+	{
+		cpu = emu->global_cpu[i];
+		row = cpu->gindex + 1;
+		ucpu = &emu->cpu_chan;
+
+		chan_cpu_init(cpu, ucpu, CHAN_KERNEL_CS, CHAN_TRACK_TH_ACTIVE, 0, 0, 1, row, prv_cpu, clock);
+	}
+}
+
+/* --------------------------- pre ------------------------------- */
+
+static void
+context_switch(struct ovni_emu *emu)
+{
+	struct ovni_ethread *th;
+	struct ovni_chan *chan;
+
+	th = emu->cur_thread;
+	chan = &th->chan[CHAN_KERNEL_CS];
+
+	switch(emu->cur_ev->header.value)
+	{
+		case 'O':
+			chan_push(chan, ST_KERNEL_CSOUT);
+			break;
+		case 'I':
+			chan_pop(chan, ST_KERNEL_CSOUT);
+			break;
+		default:
+			err("unexpected value '%c' (expecting 'O' or 'I')\n",
+					emu->cur_ev->header.value);
+			abort();
+	}
+}
+
+void
+hook_pre_kernel(struct ovni_emu *emu)
+{
+	assert(emu->cur_ev->header.model == 'K');
+
+	switch(emu->cur_ev->header.category)
+	{
+		case 'C': context_switch(emu); break;
+		default:
+			break;
+	}
+}

pcf.c

@@ -383,6 +383,27 @@ static const struct event_type thread_nanos6_mode = {
 	nanos6_mode_values
 };
 
+/* ---------------- CHAN_KERNEL_CS ---------------- */
+
+struct event_value kernel_cs_values[] = {
+	{ ST_NULL,		"NULL" },
+	{ ST_TOO_MANY_TH,	"Kernel CS: Unknown, multiple threads running" },
+	{ ST_KERNEL_CSOUT,	"Context switch: Out of the CPU" },
+	{ -1, NULL },
+};
+
+struct event_type cpu_kernel_cs = {
+	0, CHAN_KERNEL_CS, CHAN_CPU,
+	"CPU: Context switches of the ACTIVE thread",
+	kernel_cs_values
+};
+
+struct event_type thread_kernel_cs = {
+	0, CHAN_KERNEL_CS, CHAN_TH,
+	"Thread: Context switches of the CURRENT thread",
+	kernel_cs_values
+};
+
 /* ----------------------------------------------- */
 
 static void
@@ -475,6 +496,7 @@ write_events(FILE *f, struct ovni_emu *emu)
 	write_event_type(f, &thread_tampi_mode);
 	write_event_type(f, &thread_openmp_mode);
 	write_event_type(f, &thread_nanos6_mode);
+	write_event_type(f, &thread_kernel_cs);
 
 	/* CPU */
 	write_event_type(f, &cpu_ovni_pid);
@@ -491,6 +513,7 @@ write_events(FILE *f, struct ovni_emu *emu)
 	write_event_type(f, &cpu_tampi_mode);
 	write_event_type(f, &cpu_openmp_mode);
 	write_event_type(f, &cpu_nanos6_mode);
+	write_event_type(f, &cpu_kernel_cs);
 
 	/* Custom */
 	write_cpu_type(f, &thread_cpu_affinity, emu);