From 1b95fa813b2aed48bec40ad8505389c9b4851971 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Date: Tue, 10 Sep 2024 10:30:03 +0200 Subject: [PATCH] Store process information in thread metadata --- src/emu/metadata.c | 27 ++------------- src/emu/metadata.h | 4 +-- src/emu/proc.c | 82 ++++++++++------------------------------------ src/emu/proc.h | 7 ++-- src/emu/stream.c | 2 ++ src/emu/system.c | 29 +++++----------- src/rt/ovni.c | 32 ++++++++++++------ 7 files changed, 60 insertions(+), 123 deletions(-) diff --git a/src/emu/metadata.c b/src/emu/metadata.c index d0c8a90..0cf63c1 100644 --- a/src/emu/metadata.c +++ b/src/emu/metadata.c @@ -9,26 +9,9 @@ #include "ovni.h" #include "parson.h" #include "proc.h" +#include "stream.h" #include "thread.h" -static JSON_Object * -load_json(const char *path) -{ - JSON_Value *vmeta = json_parse_file_with_comments(path); - if (vmeta == NULL) { - err("json_parse_file_with_comments() failed"); - return NULL; - } - - JSON_Object *meta = json_value_get_object(vmeta); - if (meta == NULL) { - err("json_value_get_object() failed"); - return NULL; - } - - return meta; -} - static int check_version(JSON_Object *meta) { @@ -109,13 +92,9 @@ load_cpus(struct loom *loom, JSON_Object *meta) } int -metadata_load_proc(const char *path, struct loom *loom, struct proc *proc) +metadata_load_proc(struct stream *s, struct loom *loom, struct proc *proc) { - JSON_Object *meta = load_json(path); - if (meta == NULL) { - err("cannot load proc metadata from file %s", path); - return -1; - } + JSON_Object *meta = stream_metadata(s); if (check_version(meta) != 0) { err("version check failed"); diff --git a/src/emu/metadata.h b/src/emu/metadata.h index 5771885..2e1c3e7 100644 --- a/src/emu/metadata.h +++ b/src/emu/metadata.h @@ -5,10 +5,10 @@ #define METADATA_H #include "common.h" +struct stream; struct loom; struct proc; -struct thread; -USE_RET int metadata_load_proc(const char *path, struct loom *loom, struct proc *proc); +USE_RET int metadata_load_proc(struct stream *s, struct loom *loom, struct proc *proc); #endif /* METADATA_H */ diff --git a/src/emu/proc.c b/src/emu/proc.c index 7e4f363..f42e396 100644 --- a/src/emu/proc.c +++ b/src/emu/proc.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2021-2023 Barcelona Supercomputing Center (BSC) +/* Copyright (c) 2021-2024 Barcelona Supercomputing Center (BSC) * SPDX-License-Identifier: GPL-3.0-or-later */ #include "proc.h" @@ -6,85 +6,39 @@ #include #include #include "path.h" +#include "stream.h" #include "thread.h" -static int -get_pid(const char *id, int *pid) +int +proc_stream_get_pid(struct stream *s) { - /* TODO: Store the PID the metadata.json instead */ + JSON_Object *meta = stream_metadata(s); - /* The id must be like "loom.host01.123/proc.345" */ - if (path_count(id, '/') != 1) { - err("proc id can only contain one '/': %s", id); + double pid = json_object_dotget_number(meta, "ovni.pid"); + + /* Zero is used for errors, so forbidden for pid too */ + if (pid == 0) { + err("cannot get attribute ovni.pid for stream: %s", + s->relpath); return -1; } - /* Get the proc.345 part */ - const char *procname; - if (path_next(id, '/', &procname) != 0) { - err("cannot get proc name"); - return -1; - } - - /* Ensure the prefix is ok */ - const char prefix[] = "proc."; - if (!path_has_prefix(procname, prefix)) { - err("proc name must start with '%s': %s", prefix, id); - return -1; - } - - /* Get the 345 part */ - const char *pidstr; - if (path_next(procname, '.', &pidstr) != 0) { - err("cannot find proc dot in '%s'", id); - return -1; - } - - *pid = atoi(pidstr); - - return 0; + return (int) pid; } int -proc_relpath_get_pid(const char *relpath, int *pid) -{ - char id[PATH_MAX]; - - if (snprintf(id, PATH_MAX, "%s", relpath) >= PATH_MAX) { - err("path too long"); - return -1; - } - - if (path_keep(id, 2) != 0) { - err("cannot delimite proc dir"); - return -1; - } - - return get_pid(id, pid); -} - -int -proc_init_begin(struct proc *proc, const char *relpath) +proc_init_begin(struct proc *proc, int pid) { memset(proc, 0, sizeof(struct proc)); proc->gindex = -1; + proc->pid = pid; - if (snprintf(proc->id, PATH_MAX, "%s", relpath) >= PATH_MAX) { + if (snprintf(proc->id, PATH_MAX, "proc.%d", pid) >= PATH_MAX) { err("path too long"); return -1; } - if (path_keep(proc->id, 2) != 0) { - err("cannot delimite proc dir"); - return -1; - } - - if (get_pid(proc->id, &proc->pid) != 0) { - err("cannot parse proc pid"); - return -1; - } - dbg("created proc %s", proc->id); return 0; @@ -118,15 +72,15 @@ proc_load_metadata(struct proc *proc, JSON_Object *meta) proc->metadata_version = (int) json_number(version_val); - JSON_Value *appid_val = json_object_get_value(meta, "app_id"); + JSON_Value *appid_val = json_object_dotget_value(meta, "ovni.app_id"); if (appid_val == NULL) { - err("missing attribute 'app_id' in metadata"); + err("missing attribute 'ovni.app_id' in metadata"); return -1; } proc->appid = (int) json_number(appid_val); - JSON_Value *rank_val = json_object_get_value(meta, "rank"); + JSON_Value *rank_val = json_object_dotget_value(meta, "ovni.rank"); if (rank_val != NULL) proc->rank = (int) json_number(rank_val); diff --git a/src/emu/proc.h b/src/emu/proc.h index e60113b..01c0fd2 100644 --- a/src/emu/proc.h +++ b/src/emu/proc.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2021-2023 Barcelona Supercomputing Center (BSC) +/* Copyright (c) 2021-2024 Barcelona Supercomputing Center (BSC) * SPDX-License-Identifier: GPL-3.0-or-later */ #ifndef PROC_H @@ -11,6 +11,7 @@ #include "parson.h" #include "uthash.h" struct loom; +struct stream; struct thread; struct proc { @@ -45,8 +46,8 @@ struct proc { struct extend ext; }; -USE_RET int proc_relpath_get_pid(const char *relpath, int *pid); -USE_RET int proc_init_begin(struct proc *proc, const char *id); +USE_RET int proc_stream_get_pid(struct stream *s); +USE_RET int proc_init_begin(struct proc *proc, int pid); USE_RET int proc_init_end(struct proc *proc); USE_RET int proc_get_pid(struct proc *proc); void proc_set_gindex(struct proc *proc, int64_t gindex); diff --git a/src/emu/stream.c b/src/emu/stream.c index 04aba58..d0bee32 100644 --- a/src/emu/stream.c +++ b/src/emu/stream.c @@ -128,6 +128,8 @@ load_json(const char *path) return NULL; } + /* TODO: Check version */ + return meta; } diff --git a/src/emu/system.c b/src/emu/system.c index e09ab2c..247414b 100644 --- a/src/emu/system.c +++ b/src/emu/system.c @@ -64,11 +64,11 @@ create_thread(struct proc *proc, struct stream *s) } static struct proc * -create_proc(struct loom *loom, const char *tracedir, const char *relpath) +create_proc(struct loom *loom, struct stream *s) { - int pid; - if (proc_relpath_get_pid(relpath, &pid) != 0) { - err("cannot get proc pid from %s", relpath); + int pid = proc_stream_get_pid(s); + if (pid < 0) { + err("cannot get proc pid from stream: %s", s->relpath); return NULL; } @@ -84,23 +84,14 @@ create_proc(struct loom *loom, const char *tracedir, const char *relpath) return NULL; } - if (proc_init_begin(proc, relpath) != 0) { - err("proc_init_begin failed"); - return NULL; - } - - /* Build metadata path */ - char mpath[PATH_MAX]; - - if (snprintf(mpath, PATH_MAX, "%s/%s/metadata.json", - tracedir, proc->id) >= PATH_MAX) { - err("path too long"); + if (proc_init_begin(proc, pid) != 0) { + err("proc_init_begin failed: %s", s->relpath); return NULL; } /* Load metadata too */ - if (metadata_load_proc(mpath, loom, proc) != 0) { - err("cannot load metadata from %s", mpath); + if (metadata_load_proc(s, loom, proc) != 0) { + err("cannot load metadata from %s", s->relpath); return NULL; } @@ -235,8 +226,6 @@ is_thread_stream(struct stream *s) static int create_system(struct system *sys, struct trace *trace) { - const char *dir = trace->tracedir; - /* Allocate the lpt map */ sys->lpt = calloc((size_t) trace->nstreams, sizeof(struct lpt)); if (sys->lpt == NULL) { @@ -262,7 +251,7 @@ create_system(struct system *sys, struct trace *trace) } /* Loads metadata too */ - struct proc *proc = create_proc(loom, dir, s->relpath); + struct proc *proc = create_proc(loom, s); if (proc == NULL) { err("create_proc failed"); return -1; diff --git a/src/rt/ovni.c b/src/rt/ovni.c index cccad79..e459138 100644 --- a/src/rt/ovni.c +++ b/src/rt/ovni.c @@ -56,6 +56,9 @@ struct ovni_rproc { char loom[OVNI_MAX_HOSTNAME]; int ncpus; clockid_t clockid; + int rank_set; + int rank; + int nranks; int ready; @@ -258,16 +261,9 @@ ovni_proc_set_rank(int rank, int nranks) if (!rproc.ready) die("process not yet initialized"); - JSON_Object *meta = json_value_get_object(rproc.meta); - - if (meta == NULL) - die("json_value_get_object failed"); - - if (json_object_set_number(meta, "rank", rank) != 0) - die("json_object_set_number for rank failed"); - - if (json_object_set_number(meta, "nranks", nranks) != 0) - die("json_object_set_number for nranks failed"); + rproc.rank_set = 1; + rproc.rank = rank; + rproc.nranks = nranks; } /* Create $tracedir/loom.$loom/proc.$pid and return it in path. */ @@ -559,6 +555,9 @@ thread_metadata_populate(void) if (json_object_dotset_string(meta, "ovni.loom", rproc.loom) != 0) die("json_object_dotset_string failed"); + + if (json_object_dotset_number(meta, "ovni.app_id", rproc.app) != 0) + die("json_object_dotset_number for ovni.app_id failed"); } static void @@ -610,6 +609,16 @@ ovni_thread_init(pid_t tid) rthread.ready = 1; } +static void +set_thread_rank(JSON_Object *meta) +{ + if (json_object_dotset_number(meta, "ovni.rank", rproc.rank) != 0) + die("json_object_set_number for rank failed"); + + if (json_object_dotset_number(meta, "ovni.nranks", rproc.nranks) != 0) + die("json_object_set_number for nranks failed"); +} + void ovni_thread_free(void) { @@ -624,6 +633,9 @@ ovni_thread_free(void) if (meta == NULL) die("json_value_get_object failed"); + if (rproc.rank_set) + set_thread_rank(meta); + /* Mark it finished so we can detect partial streams */ if (json_object_dotset_number(meta, "ovni.finished", 1) != 0) die("json_object_dotset_string failed");