Store process information in thread metadata

src/emu/metadata.c

@@ -9,26 +9,9 @@
 #include "ovni.h"
 #include "parson.h"
 #include "proc.h"
+#include "stream.h"
 #include "thread.h"
 
-static JSON_Object *
-load_json(const char *path)
-{
-	JSON_Value *vmeta = json_parse_file_with_comments(path);
-	if (vmeta == NULL) {
-		err("json_parse_file_with_comments() failed");
-		return NULL;
-	}
-
-	JSON_Object *meta = json_value_get_object(vmeta);
-	if (meta == NULL) {
-		err("json_value_get_object() failed");
-		return NULL;
-	}
-
-	return meta;
-}
-
 static int
 check_version(JSON_Object *meta)
 {
@@ -109,13 +92,9 @@ load_cpus(struct loom *loom, JSON_Object *meta)
 }
 
 int
-metadata_load_proc(const char *path, struct loom *loom, struct proc *proc)
+metadata_load_proc(struct stream *s, struct loom *loom, struct proc *proc)
 {
-	JSON_Object *meta = load_json(path);
+	JSON_Object *meta = stream_metadata(s);
-	if (meta == NULL) {
-		err("cannot load proc metadata from file %s", path);
-		return -1;
-	}
 
 	if (check_version(meta) != 0) {
 		err("version check failed");

src/emu/metadata.h

@@ -5,10 +5,10 @@
 #define METADATA_H
 
 #include "common.h"
+struct stream;
 struct loom;
 struct proc;
-struct thread;
 
-USE_RET int metadata_load_proc(const char *path, struct loom *loom, struct proc *proc);
+USE_RET int metadata_load_proc(struct stream *s, struct loom *loom, struct proc *proc);
 
 #endif /* METADATA_H */

src/emu/proc.c

@@ -1,4 +1,4 @@
-/* Copyright (c) 2021-2023 Barcelona Supercomputing Center (BSC)
+/* Copyright (c) 2021-2024 Barcelona Supercomputing Center (BSC)
  * SPDX-License-Identifier: GPL-3.0-or-later */
 
 #include "proc.h"
@@ -6,85 +6,39 @@
 #include <stdlib.h>
 #include <string.h>
 #include "path.h"
+#include "stream.h"
 #include "thread.h"
 
-static int
+int
-get_pid(const char *id, int *pid)
+proc_stream_get_pid(struct stream *s)
 {
-	/* TODO: Store the PID the metadata.json instead */
+	JSON_Object *meta = stream_metadata(s);
 
-	/* The id must be like "loom.host01.123/proc.345" */
+	double pid = json_object_dotget_number(meta, "ovni.pid");
-	if (path_count(id, '/') != 1) {
+
-		err("proc id can only contain one '/': %s", id);
+	/* Zero is used for errors, so forbidden for pid too */
+	if (pid == 0) {
+		err("cannot get attribute ovni.pid for stream: %s",
+				s->relpath);
 		return -1;
 	}
 
-	/* Get the proc.345 part */
+	return (int) pid;
-	const char *procname;
-	if (path_next(id, '/', &procname) != 0) {
-		err("cannot get proc name");
-		return -1;
-	}
-
-	/* Ensure the prefix is ok */
-	const char prefix[] = "proc.";
-	if (!path_has_prefix(procname, prefix)) {
-		err("proc name must start with '%s': %s", prefix, id);
-		return -1;
-	}
-
-	/* Get the 345 part */
-	const char *pidstr;
-	if (path_next(procname, '.', &pidstr) != 0) {
-		err("cannot find proc dot in '%s'", id);
-		return -1;
-	}
-
-	*pid = atoi(pidstr);
-
-	return 0;
 }
 
 int
-proc_relpath_get_pid(const char *relpath, int *pid)
+proc_init_begin(struct proc *proc, int pid)
-{
-	char id[PATH_MAX];
-
-	if (snprintf(id, PATH_MAX, "%s", relpath) >= PATH_MAX) {
-		err("path too long");
-		return -1;
-	}
-
-	if (path_keep(id, 2) != 0) {
-		err("cannot delimite proc dir");
-		return -1;
-	}
-
-	return get_pid(id, pid);
-}
-
-int
-proc_init_begin(struct proc *proc, const char *relpath)
 {
 	memset(proc, 0, sizeof(struct proc));
 
 	proc->gindex = -1;
+	proc->pid = pid;
 
-	if (snprintf(proc->id, PATH_MAX, "%s", relpath) >= PATH_MAX) {
+	if (snprintf(proc->id, PATH_MAX, "proc.%d", pid) >= PATH_MAX) {
 		err("path too long");
 		return -1;
 	}
 
-	if (path_keep(proc->id, 2) != 0) {
-		err("cannot delimite proc dir");
-		return -1;
-	}
-
-	if (get_pid(proc->id, &proc->pid) != 0) {
-		err("cannot parse proc pid");
-		return -1;
-	}
-
 	dbg("created proc %s", proc->id);
 
 	return 0;
@@ -118,15 +72,15 @@ proc_load_metadata(struct proc *proc, JSON_Object *meta)
 
 	proc->metadata_version = (int) json_number(version_val);
 
-	JSON_Value *appid_val = json_object_get_value(meta, "app_id");
+	JSON_Value *appid_val = json_object_dotget_value(meta, "ovni.app_id");
 	if (appid_val == NULL) {
-		err("missing attribute 'app_id' in metadata");
+		err("missing attribute 'ovni.app_id' in metadata");
 		return -1;
 	}
 
 	proc->appid = (int) json_number(appid_val);
 
-	JSON_Value *rank_val = json_object_get_value(meta, "rank");
+	JSON_Value *rank_val = json_object_dotget_value(meta, "ovni.rank");
 
 	if (rank_val != NULL)
 		proc->rank = (int) json_number(rank_val);

src/emu/proc.h

@@ -1,4 +1,4 @@
-/* Copyright (c) 2021-2023 Barcelona Supercomputing Center (BSC)
+/* Copyright (c) 2021-2024 Barcelona Supercomputing Center (BSC)
  * SPDX-License-Identifier: GPL-3.0-or-later */
 
 #ifndef PROC_H
@@ -11,6 +11,7 @@
 #include "parson.h"
 #include "uthash.h"
 struct loom;
+struct stream;
 struct thread;
 
 struct proc {
@@ -45,8 +46,8 @@ struct proc {
 	struct extend ext;
 };
 
-USE_RET int proc_relpath_get_pid(const char *relpath, int *pid);
+USE_RET int proc_stream_get_pid(struct stream *s);
-USE_RET int proc_init_begin(struct proc *proc, const char *id);
+USE_RET int proc_init_begin(struct proc *proc, int pid);
 USE_RET int proc_init_end(struct proc *proc);
 USE_RET int proc_get_pid(struct proc *proc);
         void proc_set_gindex(struct proc *proc, int64_t gindex);

src/emu/stream.c

@@ -128,6 +128,8 @@ load_json(const char *path)
 		return NULL;
 	}
 
+	/* TODO: Check version */
+
 	return meta;
 }
 

src/emu/system.c

@@ -64,11 +64,11 @@ create_thread(struct proc *proc, struct stream *s)
 }
 
 static struct proc *
-create_proc(struct loom *loom, const char *tracedir, const char *relpath)
+create_proc(struct loom *loom, struct stream *s)
 {
-	int pid;
+	int pid = proc_stream_get_pid(s);
-	if (proc_relpath_get_pid(relpath, &pid) != 0) {
+	if (pid < 0) {
-		err("cannot get proc pid from %s", relpath);
+		err("cannot get proc pid from stream: %s", s->relpath);
 		return NULL;
 	}
 
@@ -84,23 +84,14 @@ create_proc(struct loom *loom, const char *tracedir, const char *relpath)
 		return NULL;
 	}
 
-	if (proc_init_begin(proc, relpath) != 0) {
+	if (proc_init_begin(proc, pid) != 0) {
-		err("proc_init_begin failed");
+		err("proc_init_begin failed: %s", s->relpath);
-		return NULL;
-	}
-
-	/* Build metadata path */
-	char mpath[PATH_MAX];
-
-	if (snprintf(mpath, PATH_MAX, "%s/%s/metadata.json",
-				tracedir, proc->id) >= PATH_MAX) {
-		err("path too long");
 		return NULL;
 	}
 
 	/* Load metadata too */
-	if (metadata_load_proc(mpath, loom, proc) != 0) {
+	if (metadata_load_proc(s, loom, proc) != 0) {
-		err("cannot load metadata from %s", mpath);
+		err("cannot load metadata from %s", s->relpath);
 		return NULL;
 	}
 
@@ -235,8 +226,6 @@ is_thread_stream(struct stream *s)
 static int
 create_system(struct system *sys, struct trace *trace)
 {
-	const char *dir = trace->tracedir;
-
 	/* Allocate the lpt map */
 	sys->lpt = calloc((size_t) trace->nstreams, sizeof(struct lpt));
 	if (sys->lpt == NULL) {
@@ -262,7 +251,7 @@ create_system(struct system *sys, struct trace *trace)
 		}
 
 		/* Loads metadata too */
-		struct proc *proc = create_proc(loom, dir, s->relpath);
+		struct proc *proc = create_proc(loom, s);
 		if (proc == NULL) {
 			err("create_proc failed");
 			return -1;

src/rt/ovni.c

@@ -56,6 +56,9 @@ struct ovni_rproc {
 	char loom[OVNI_MAX_HOSTNAME];
 	int ncpus;
 	clockid_t clockid;
+	int rank_set;
+	int rank;
+	int nranks;
 
 	int ready;
 
@@ -258,16 +261,9 @@ ovni_proc_set_rank(int rank, int nranks)
 	if (!rproc.ready)
 		die("process not yet initialized");
 
-	JSON_Object *meta = json_value_get_object(rproc.meta);
+	rproc.rank_set = 1;
-
+	rproc.rank = rank;
-	if (meta == NULL)
+	rproc.nranks = nranks;
-		die("json_value_get_object failed");
-
-	if (json_object_set_number(meta, "rank", rank) != 0)
-		die("json_object_set_number for rank failed");
-
-	if (json_object_set_number(meta, "nranks", nranks) != 0)
-		die("json_object_set_number for nranks failed");
 }
 
 /* Create $tracedir/loom.$loom/proc.$pid and return it in path. */
@@ -559,6 +555,9 @@ thread_metadata_populate(void)
 
 	if (json_object_dotset_string(meta, "ovni.loom", rproc.loom) != 0)
 		die("json_object_dotset_string failed");
+
+	if (json_object_dotset_number(meta, "ovni.app_id", rproc.app) != 0)
+		die("json_object_dotset_number for ovni.app_id failed");
 }
 
 static void
@@ -610,6 +609,16 @@ ovni_thread_init(pid_t tid)
 	rthread.ready = 1;
 }
 
+static void
+set_thread_rank(JSON_Object *meta)
+{
+	if (json_object_dotset_number(meta, "ovni.rank", rproc.rank) != 0)
+		die("json_object_set_number for rank failed");
+
+	if (json_object_dotset_number(meta, "ovni.nranks", rproc.nranks) != 0)
+		die("json_object_set_number for nranks failed");
+}
+
 void
 ovni_thread_free(void)
 {
@@ -624,6 +633,9 @@ ovni_thread_free(void)
 	if (meta == NULL)
 		die("json_value_get_object failed");
 
+	if (rproc.rank_set)
+		set_thread_rank(meta);
+
 	/* Mark it finished so we can detect partial streams */
 	if (json_object_dotset_number(meta, "ovni.finished", 1) != 0)
 		die("json_object_dotset_string failed");