Add support for HWC in nOS-V

This commit is contained in:
Rodrigo Arias 2025-07-25 13:53:39 +02:00
parent e0871bd391
commit a164642653
13 changed files with 674 additions and 8 deletions

View File

@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Added
- Add support for hardware counters (HWC) in nOS-V.
### Changed
- Increase nOS-V model version to 2.6.0.
### Fixed
- Fix a bug in ovniemu when loading loom CPUs from multiple threads.

View File

@ -1,7 +1,7 @@
# Emulator events
This is a exhaustive list of the events recognized by the emulator.
Built on May 2 2025.
Built on Jul 28 2025.
## Model nanos6
@ -633,7 +633,7 @@ List of events for the model *tampi* with identifier **`T`** at version `1.0.0`:
## Model nosv
List of events for the model *nosv* with identifier **`V`** at version `2.5.1`:
List of events for the model *nosv* with identifier **`V`** at version `2.6.0`:
<dl>
<dt><a id="VTc" href="#VTc"><pre>VTc(u32 taskid, u32 typeid)</pre></a></dt>
<dd>creates task %{taskid} with type %{typeid}</dd>
@ -761,4 +761,6 @@ List of events for the model *nosv* with identifier **`V`** at version `2.5.1`:
<dd>sets progress state to Resting</dd>
<dt><a id="VPa" href="#VPa"><pre>VPa</pre></a></dt>
<dd>sets progress state to Absorbing</dd>
<dt><a id="VWC" href="#VWC"><pre>VWC+(i64 value)</pre></a></dt>
<dd>set hardware counters (first %{value})</dd>
</dl>

View File

@ -45,6 +45,8 @@ Track changes in emulator model versions.
## nOS-V
- nosv 2.6.0
- Add support for hardware counters event `VWC`.
- nosv 2.5.1
- Remove task complete callback from between `VTx` and `VTe` events.
- nosv 2.5.0

View File

@ -54,8 +54,9 @@ add_library(emu STATIC
nanos6/event.c
nanos6/breakdown.c
nosv/breakdown.c
nosv/setup.c
nosv/event.c
nosv/hwc.c
nosv/setup.c
nodes/setup.c
nodes/event.c
mpi/setup.c

View File

@ -38,7 +38,9 @@ enum emu_prv_types {
PRV_OPENMP_BREAKDOWN = 53,
PRV_OVNI_MARK = 100,
/* User marks [100, 200) */
PRV_RESERVED = 200,
PRV_NOSV_HWC = 200,
/* nOS-V HWC [200, 300) */
PRV_RESERVED = 300,
};
#endif /* EMU_PRV_H */

View File

@ -596,6 +596,8 @@ process_ev(struct emu *emu)
return pre_task(emu);
case 'Y':
return pre_type(emu);
case 'W':
return hwc_event(emu);
default:
err("unknown nOS-V event category");
return -1;

545
src/emu/nosv/hwc.c Normal file
View File

@ -0,0 +1,545 @@
#include "hwc.h"
#include "chan.h"
#include "cpu.h"
#include "emu.h"
#include "emu_ev.h"
#include "emu_prv.h"
#include "inttypes.h"
#include "nosv_priv.h"
#include "ovni.h"
#include "parson.h"
#include "proc.h"
#include "pv/cfg_file.h"
#include "pv/pcf.h"
#include "pv/prv.h"
#include "pv/pvt.h"
#include "thread.h"
#include "track.h"
#include "uthash.h"
#include <errno.h>
static int
parse_hwc(struct nosv_hwc_emu *hwc_emu, const char *indexstr, JSON_Value *hwcval)
{
errno = 0;
char *endptr = NULL;
size_t index = (size_t) strtol(indexstr, &endptr, 10);
if (errno != 0 || endptr == indexstr || endptr[0] != '\0') {
err("failed to parse hwc index: %s", indexstr);
return -1;
}
if (index >= 100) {
err("hwc index should be in [0, 100) range: %zd", index);
return -1;
}
if (index >= hwc_emu->n) {
err("hwc index %zd exceeds allocated counters %zd",
index, hwc_emu->n);
return -1;
}
JSON_Object *hwc = json_value_get_object(hwcval);
if (hwc == NULL) {
err("json_value_get_object() failed");
return -1;
}
const char *name = json_object_get_string(hwc, "name");
if (name == NULL) {
err("json_object_get_string() for name failed");
return -1;
}
if (hwc_emu->name[index] == NULL) {
hwc_emu->name[index] = strdup(name);
if (hwc_emu->name[index] == NULL) {
err("strdup failed");
return -1;
}
dbg("got hwc with index %zd and name %s", index, hwc_emu->name[index]);
} else if (strcmp(hwc_emu->name[index], name) != 0) {
err("hwc at %zd already defined as %s",
index, hwc_emu->name[index]);
return -1;
}
return 0;
}
static int
scan_thread(struct nosv_hwc_emu *hwc_emu, struct thread *t)
{
JSON_Object *obj = json_object_dotget_object(t->meta, "nosv.hwc");
/* No HWC in this thread */
if (obj == NULL)
return 0;
size_t n = json_object_get_count(obj);
/* Ignore empty dictionaries */
if (n == 0)
return 0;
if (hwc_emu->n == 0) {
hwc_emu->name = calloc(n, sizeof(char *));
hwc_emu->n = n;
} else if (hwc_emu->n != n) {
/* We have a mismatch */
err("thread %s defines %zd hardware counters, but emu already has %zd",
t->id, n, hwc_emu->n);
return -1;
}
for (size_t i = 0; i < n; i++) {
const char *indexstr = json_object_get_name(obj, i);
if (indexstr == NULL) {
err("json_object_get_name failed");
return -1;
}
JSON_Value *hwcval = json_object_get_value_at(obj, i);
if (hwcval == NULL) {
err("json_object_get_value_at failed");
return -1;
}
if (parse_hwc(hwc_emu, indexstr, hwcval) != 0) {
err("cannot parse HWC");
return -1;
}
}
return 0;
}
static int
create_thread_chan(struct nosv_hwc_emu *emu, struct bay *bay, struct thread *th)
{
struct nosv_thread *nosv_thread = EXT(th, 'V');
struct nosv_hwc_thread *t = &nosv_thread->hwc;
/* Create as many channels as required */
t->chan = calloc(emu->n, sizeof(struct chan));
if (t->chan == NULL) {
err("calloc failed:");
return -1;
}
t->n = emu->n;
for (size_t i = 0; i < t->n; i++) {
struct chan *ch = &t->chan[i];
chan_init(ch, CHAN_SINGLE, "nosv.thread%"PRIi64".hwc%zd",
th->gindex, i);
/* Allow duplicates, we can emit the same HWC value twice */
chan_prop_set(ch, CHAN_ALLOW_DUP, 1);
if (bay_register(bay, ch) != 0) {
err("bay_register failed");
return -1;
}
}
/* Setup tracking */
t->track = calloc(t->n, sizeof(struct track));
if (t->track == NULL) {
err("calloc failed:");
return -1;
}
for (size_t i = 0; i < t->n; i++) {
struct track *track = &t->track[i];
/* For now only tracking to active thread is supported */
if (track_init(track, bay, TRACK_TYPE_TH, TRACK_TH_ACT,
"nosv.thread%"PRIi64".hwc%zd",
th->gindex, i) != 0) {
err("track_init failed");
return -1;
}
}
return 0;
}
static int
init_cpu(struct nosv_hwc_emu *emu, struct bay *bay, struct cpu *cpu)
{
struct nosv_cpu *nosv_cpu = EXT(cpu, 'V');
struct nosv_hwc_cpu *c = &nosv_cpu->hwc;
/* Setup tracking */
c->track = calloc(emu->n, sizeof(struct track));
if (c->track == NULL) {
err("calloc failed:");
return -1;
}
c->n = emu->n;
for (size_t i = 0; i < c->n; i++) {
struct track *track = &c->track[i];
/* For now only tracking to running thread is supported */
if (track_init(track, bay, TRACK_TYPE_TH, TRACK_TH_RUN,
"nosv.cpu%"PRIi64".hwc%zd",
cpu->gindex, i) != 0) {
err("track_init failed");
return -1;
}
}
return 0;
}
int
hwc_create(struct emu *emu)
{
struct nosv_emu *nosv_emu = EXT(emu, 'V');
struct nosv_hwc_emu *hwc_emu = &nosv_emu->hwc;
for (struct thread *th = emu->system.threads; th; th = th->gnext) {
if (scan_thread(hwc_emu, th) != 0) {
err("scan_thread failed");
return -1;
}
}
/* Early exit if no counters found */
if (hwc_emu->n == 0) {
dbg("no hwc counters found");
return 0;
}
/* Create a buffer to do aligned reads */
hwc_emu->values = calloc(hwc_emu->n, sizeof(int64_t));
if (hwc_emu->values == NULL) {
err("calloc failed:");
return -1;
}
/* Once we know how many HWC we have, allocate the channels for threads
* and CPUs. */
for (struct thread *th = emu->system.threads; th; th = th->gnext) {
if (create_thread_chan(hwc_emu, &emu->bay, th) != 0) {
err("create_thread_chan failed");
return -1;
}
}
for (struct cpu *cpu = emu->system.cpus; cpu; cpu = cpu->next) {
if (init_cpu(hwc_emu, &emu->bay, cpu) != 0) {
err("init_cpu failed");
return -1;
}
}
return 0;
}
static int
connect_thread_prv(struct bay *bay, struct thread *systh, struct prv *prv)
{
struct nosv_thread *nosv_thread = EXT(systh, 'V');
struct nosv_hwc_thread *t = &nosv_thread->hwc;
for (size_t i = 0; i < t->n; i++) {
struct track *track = &t->track[i];
struct chan *ch = &t->chan[i];
struct chan *sel = &systh->chan[TH_CHAN_STATE];
/* Connect the input and sel channel to the mux */
if (track_connect_thread(track, ch, sel, 1) != 0) {
err("track_connect_thread failed");
return -1;
}
/* Then connect the output of the tracking module to the prv
* trace for the current thread */
struct chan *out = track_get_output(track);
long row = (long) systh->gindex;
long flags = PRV_SKIPDUPNULL | PRV_ZERO;
long prvtype = (long) (PRV_NOSV_HWC + i);
if (prv_register(prv, row, prvtype, bay, out, flags)) {
err("prv_register failed");
return -1;
}
}
return 0;
}
static int
init_pcf(struct emu *emu, struct pcf *pcf)
{
struct nosv_emu *nosv_emu = EXT(emu, 'V');
struct nosv_hwc_emu *hwc_emu = &nosv_emu->hwc;
for (size_t i = 0; i < hwc_emu->n; i++) {
long prvtype = (long) (PRV_NOSV_HWC + i);
const char *name = hwc_emu->name[i];
struct pcf_type *pcftype = pcf_add_type(pcf, (int) prvtype, name);
if (pcftype == NULL) {
err("pcf_add_type failed");
return -1;
}
}
return 0;
}
static int
connect_thread(struct emu *emu)
{
struct pvt *pvt = recorder_find_pvt(&emu->recorder, "thread");
if (pvt == NULL) {
err("cannot find thread pvt");
return -1;
}
/* Connect thread channels to PRV */
struct prv *prv = pvt_get_prv(pvt);
for (struct thread *t = emu->system.threads; t; t = t->gnext) {
if (connect_thread_prv(&emu->bay, t, prv) != 0) {
err("connect_thread_prv failed");
return -1;
}
}
/* Init thread PCF */
struct pcf *pcf = pvt_get_pcf(pvt);
if (init_pcf(emu, pcf) != 0) {
err("init_pcf failed");
return -1;
}
return 0;
}
static int
connect_cpu_prv(struct emu *emu, struct cpu *syscpu, struct prv *prv)
{
struct nosv_emu *nosv_emu = EXT(emu, 'V');
struct nosv_hwc_emu *hwc_emu = &nosv_emu->hwc;
struct nosv_cpu *nosv_cpu = EXT(syscpu, 'V');
struct nosv_hwc_cpu *hwc_cpu = &nosv_cpu->hwc;
for (size_t i = 0; i < hwc_emu->n; i++) {
struct track *track = &hwc_cpu->track[i];
struct chan *sel = cpu_get_th_chan(syscpu);
int64_t nthreads = (int64_t) emu->system.nthreads;
if (track_set_select(track, sel, NULL, nthreads) != 0) {
err("track_select failed");
return -1;
}
/* Add each thread as input */
for (struct thread *t = emu->system.threads; t; t = t->gnext) {
struct nosv_thread *nosv_thread = EXT(t, 'V');
struct nosv_hwc_thread *hwc_thread = &nosv_thread->hwc;
/* Use the input thread directly */
struct chan *inp = &hwc_thread->chan[i];
if (track_set_input(track, t->gindex, inp) != 0) {
err("track_add_input failed");
return -1;
}
}
/* Then connect the output of the tracking module to the prv
* trace for the current cpu */
struct chan *out = track_get_output(track);
long row = (long) syscpu->gindex;
long flags = PRV_SKIPDUPNULL | PRV_ZERO;
long prvtype = (long) (PRV_NOSV_HWC + i);
if (prv_register(prv, row, prvtype, &emu->bay, out, flags)) {
err("prv_register failed");
return -1;
}
}
return 0;
}
static int
connect_cpu(struct emu *emu)
{
/* Get cpu PRV */
struct pvt *pvt = recorder_find_pvt(&emu->recorder, "cpu");
if (pvt == NULL) {
err("cannot find thread pvt");
return -1;
}
/* Connect cpu channels to PRV */
struct prv *prv = pvt_get_prv(pvt);
for (struct cpu *cpu = emu->system.cpus; cpu; cpu = cpu->next) {
if (connect_cpu_prv(emu, cpu, prv) != 0) {
err("connect_cpu_prv failed");
return -1;
}
}
/* Init cpu PCF */
struct pcf *pcf = pvt_get_pcf(pvt);
if (init_pcf(emu, pcf) != 0) {
err("init_pcf failed");
return -1;
}
return 0;
}
/* Connect the channels to the output PVTs */
int
hwc_connect(struct emu *emu)
{
struct nosv_emu *nosv_emu = EXT(emu, 'V');
struct nosv_hwc_emu *hwc_emu = &nosv_emu->hwc;
/* No HWC, so nothing to connect */
if (hwc_emu->n == 0)
return 0;
if (connect_thread(emu) != 0) {
err("connect_thread() failed");
return -1;
}
if (connect_cpu(emu) != 0) {
err("connect_cpu() failed");
return -1;
}
return 0;
}
static int
event_hwc_count(struct emu *emu)
{
struct nosv_emu *nosv_emu = EXT(emu, 'V');
struct nosv_hwc_emu *hwc_emu = &nosv_emu->hwc;
if (!emu->ev->is_jumbo) {
err("expecting a jumbo event");
return -1;
}
/* Make sure size matches */
size_t array_size = (size_t) emu->ev->payload->jumbo.size;
size_t expected_size = hwc_emu->n * sizeof(int64_t);
if (array_size != expected_size) {
err("unexpected hwc event with jumbo payload size %zd (expecting %zd)",
array_size, expected_size);
return -1;
}
/* Use memcpy to align array */
memcpy(hwc_emu->values, &emu->ev->payload->jumbo.data[0], array_size);
/* Update all HWC channels for the given thread */
for (size_t i = 0; i < hwc_emu->n; i++) {
struct nosv_thread *nosv_thread = EXT(emu->thread, 'V');
struct nosv_hwc_thread *hwc_thread = &nosv_thread->hwc;
struct chan *ch = &hwc_thread->chan[i];
if (chan_set(ch, value_int64(hwc_emu->values[i])) != 0) {
err("chan_set failed for hwc channel %s", ch->name);
return -1;
}
}
return 0;
}
int
hwc_event(struct emu *emu)
{
struct nosv_emu *nosv_emu = EXT(emu, 'V');
struct nosv_hwc_emu *hwc_emu = &nosv_emu->hwc;
/* Panic on HWC event with no counters */
if (hwc_emu->n == 0) {
err("got hwc event %s but no counters enabled", emu->ev->mcv);
return -1;
}
switch (emu->ev->v) {
case 'C':
return event_hwc_count(emu);
default:
err("unknown nosv hwc event %s", emu->ev->mcv);
return -1;
}
return 0;
}
static int
write_cfg(const char *path, size_t i, const char *fmt, const char *name)
{
char title[MAX_LABEL];
/* May truncate silently, but is safe */
snprintf(title, MAX_LABEL, fmt, name);
long type = (long) (PRV_NOSV_HWC + i);
struct cfg_file cf;
cfg_file_init(&cf, type, title);
cfg_file_color_mode(&cf, CFG_NGRAD);
cfg_file_semantic_thread(&cf, CFG_NEXT_EV_VAL);
if (cfg_file_write(&cf, path) != 0) {
err("cfg_file_write failed");
return -1;
}
return 0;
}
int
hwc_finish(struct emu *emu)
{
struct nosv_emu *nosv_emu = EXT(emu, 'V');
struct nosv_hwc_emu *hwc_emu = &nosv_emu->hwc;
if (hwc_emu->n == 0)
return 0;
/* Write CFG files with HWC names. */
for (size_t i = 0; i < hwc_emu->n; i++) {
const char *dir = emu->args.tracedir;
const char *name = hwc_emu->name[i];
char path[PATH_MAX];
/* Create thread configs */
if (snprintf(path, PATH_MAX, "%s/cfg/thread/nosv/hwc-%s.cfg", dir, name) >= PATH_MAX) {
err("hwc thread cfg path too long");
return -1;
}
if (write_cfg(path, i, "Thread: nOS-V %s of the ACTIVE thread", name) != 0) {
err("write_cfg failed");
return -1;
}
/* Same for CPUs */
if (snprintf(path, PATH_MAX, "%s/cfg/cpu/nosv/hwc-%s.cfg", dir, name) >= PATH_MAX) {
err("hwc cpu cfg path too long");
return -1;
}
if (write_cfg(path, i, "CPU: nOS-V %s of the RUNNING thread", name) != 0) {
err("write_cfg failed");
return -1;
}
}
return 0;
}

32
src/emu/nosv/hwc.h Normal file
View File

@ -0,0 +1,32 @@
#ifndef HWC_H
#define HWC_H
#include "common.h"
struct emu;
struct chan;
/* Store each HWC channel per emu */
struct nosv_hwc_emu {
char **name;
size_t n;
int64_t *values;
};
struct nosv_hwc_thread {
struct track *track;
struct chan *chan;
size_t n;
};
struct nosv_hwc_cpu {
struct track *track;
size_t n;
};
USE_RET int hwc_create(struct emu *emu);
USE_RET int hwc_connect(struct emu *emu);
USE_RET int hwc_event(struct emu *emu);
USE_RET int hwc_finish(struct emu *emu);
#endif /* HWC_H */

View File

@ -1,4 +1,4 @@
/* Copyright (c) 2021-2024 Barcelona Supercomputing Center (BSC)
/* Copyright (c) 2021-2025 Barcelona Supercomputing Center (BSC)
* SPDX-License-Identifier: GPL-3.0-or-later */
#ifndef NOSV_PRIV_H
@ -6,9 +6,10 @@
#include "breakdown.h"
#include "emu.h"
#include "task.h"
#include "hwc.h"
#include "model_cpu.h"
#include "model_thread.h"
#include "task.h"
/* Private enums */
@ -58,11 +59,13 @@ enum nosv_ss_values {
struct nosv_thread {
struct model_thread m;
struct task_stack task_stack;
struct nosv_hwc_thread hwc;
};
struct nosv_cpu {
struct model_cpu m;
struct nosv_breakdown_cpu breakdown;
struct nosv_hwc_cpu hwc;
};
struct nosv_proc {
@ -73,6 +76,7 @@ struct nosv_emu {
int connected;
int event;
struct nosv_breakdown_emu breakdown;
struct nosv_hwc_emu hwc;
};
enum nosv_progress {

View File

@ -12,6 +12,7 @@
#include "emu_prv.h"
#include "ev_spec.h"
#include "extend.h"
#include "loom.h"
#include "model.h"
#include "model_chan.h"
#include "model_cpu.h"
@ -27,6 +28,7 @@
#include "thread.h"
#include "track.h"
#include "value.h"
#include "hwc.h"
static const char model_name[] = "nosv";
enum { model_id = 'V' };
@ -81,12 +83,14 @@ static struct ev_decl model_evlist[] = {
{ "VPr", "sets progress state to Resting" },
{ "VPa", "sets progress state to Absorbing" },
{ "VWC+(i64 value)", "set hardware counters (first %{value})" },
{ NULL, NULL },
};
struct model_spec model_nosv = {
.name = model_name,
.version = "2.5.1",
.version = "2.6.0",
.evlist = model_evlist,
.model = model_id,
.create = model_nosv_create,
@ -319,6 +323,11 @@ model_nosv_create(struct emu *emu)
return -1;
}
if (hwc_create(emu) != 0) {
err("hwc_create failed");
return -1;
}
return 0;
}
@ -340,6 +349,11 @@ model_nosv_connect(struct emu *emu)
return -1;
}
if (hwc_connect(emu) != 0) {
err("hwc_connect failed");
return -1;
}
for (struct thread *th = emu->system.threads; th; th = th->gnext) {
struct nosv_thread *mth = EXT(th, model_id);
struct chan *idle = &mth->m.ch[CH_IDLE];
@ -441,6 +455,11 @@ model_nosv_finish(struct emu *emu)
return -1;
}
if (hwc_finish(emu) != 0) {
err("hwc_finish failed");
return -1;
}
/* When running in linter mode perform additional checks */
if (emu->args.linter_mode && end_lint(emu) != 0) {
err("end_lint failed");

View File

@ -38,3 +38,5 @@ test_emu(bad-nest-from-parallel.c SHOULD_FAIL
test_emu(events-from-outside-cpu.c SHOULD_FAIL
REGEX "current thread .* out of CPU")
test_emu(hwc.c)

35
test/emu/nosv/hwc.c Normal file
View File

@ -0,0 +1,35 @@
/* Copyright (c) 2025 Barcelona Supercomputing Center (BSC)
* SPDX-License-Identifier: GPL-3.0-or-later */
#include "compat.h"
#include "instr.h"
#include "instr_nosv.h"
int
main(void)
{
instr_start(0, 1);
instr_nosv_init();
enum hwc { PAPI_TOT_INS = 0, PAPI_TOT_CYC = 1, MAX_HWC };
int64_t hwc[MAX_HWC] = { 0 };
ovni_attr_set_str("nosv.hwc.0.name", "PAPI_TOT_INS");
ovni_attr_set_str("nosv.hwc.1.name", "PAPI_TOT_CYC");
instr_nosv_hwc(MAX_HWC, hwc);
for (int i = 0; i < 10; i++) {
sleep_us(100);
/* Dummy counters */
hwc[PAPI_TOT_INS] = 50 + (rand() % 100);
hwc[PAPI_TOT_CYC] = 100 + (rand() % 200);
instr_nosv_hwc(MAX_HWC, hwc);
}
instr_end();
return 0;
}

View File

@ -1,4 +1,4 @@
/* Copyright (c) 2021-2024 Barcelona Supercomputing Center (BSC)
/* Copyright (c) 2021-2025 Barcelona Supercomputing Center (BSC)
* SPDX-License-Identifier: GPL-3.0-or-later */
#ifndef INSTR_NOSV_H
@ -38,6 +38,18 @@ instr_nosv_type_create(int32_t typeid)
return task_get_type_gid(p);
}
static inline void
instr_nosv_hwc(size_t n, int64_t *counters)
{
struct ovni_ev ev = {0};
ovni_ev_set_mcv(&ev, "VWC");
ovni_ev_set_clock(&ev, (uint64_t) get_clock());
uint32_t nbytes = (uint32_t) (n * sizeof(int64_t));
ovni_ev_jumbo_emit(&ev, (uint8_t *) counters, (uint32_t) nbytes);
}
INSTR_2ARG(instr_nosv_task_create, "VTc", uint32_t, taskid, uint32_t, typeid)
INSTR_2ARG(instr_nosv_task_create_par, "VTC", uint32_t, taskid, uint32_t, typeid)
INSTR_2ARG(instr_nosv_task_execute, "VTx", uint32_t, taskid, uint32_t, bodyid)