From 676e30b4caf07494e29c2fee1bb4388d64673431 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Date: Thu, 6 Apr 2023 19:27:43 +0200 Subject: [PATCH] Order looms and processes by rank if given --- src/emu/loom.c | 67 ++++++++++++++++++++++++++++-- src/emu/loom.h | 1 + src/emu/system.c | 59 ++++++++++++++++++++++---- src/emu/system.h | 2 + test/emu/ovni/CMakeLists.txt | 2 + test/emu/ovni/sort-cpus-by-loom.c | 69 +++++++++++++++++++++++++++++++ test/emu/ovni/sort-cpus-by-rank.c | 68 ++++++++++++++++++++++++++++++ test/unit/loom.c | 1 + 8 files changed, 259 insertions(+), 10 deletions(-) create mode 100644 test/emu/ovni/sort-cpus-by-loom.c create mode 100644 test/emu/ovni/sort-cpus-by-rank.c diff --git a/src/emu/loom.c b/src/emu/loom.c index fdce75e..bdea8a5 100644 --- a/src/emu/loom.c +++ b/src/emu/loom.c @@ -152,6 +152,20 @@ by_pid(struct proc *p1, struct proc *p2) return 0; } +static int +by_rank(struct proc *p1, struct proc *p2) +{ + int id1 = p1->rank; + int id2 = p2->rank; + + if (id1 < id2) + return -1; + if (id1 > id2) + return +1; + else + return 0; +} + static int by_phyid(struct cpu *c1, struct cpu *c2) { @@ -169,12 +183,15 @@ by_phyid(struct cpu *c1, struct cpu *c2) void loom_sort(struct loom *loom) { - HASH_SORT(loom->procs, by_pid); + if (loom->rank_enabled) + HASH_SORT(loom->procs, by_rank); + else + HASH_SORT(loom->procs, by_pid); + HASH_SORT(loom->cpus, by_phyid); - for (struct proc *p = loom->procs; p; p = p->hh.next) { + for (struct proc *p = loom->procs; p; p = p->hh.next) proc_sort(p); - } } int @@ -188,6 +205,21 @@ loom_init_end(struct loom *loom) } } + /* Ensure that all processes have a rank */ + if (loom->rank_enabled) { + for (struct proc *p = loom->procs; p; p = p->hh.next) { + if (p->rank < 0) { + err("process %s has no rank information", p->id); + return -1; + } + + if (p->rank < loom->rank_min) + loom->rank_min = p->rank; + } + } else { + loom->rank_min = -1; + } + /* Populate cpus_array */ loom->cpus_array = calloc(loom->ncpus, sizeof(struct cpu *)); if (loom->cpus_array == NULL) { @@ -249,6 +281,35 @@ loom_add_proc(struct loom *loom, struct proc *proc) return -1; } + if (!proc->metadata_loaded) { + err("process %d hasn't loaded metadata", pid); + return -1; + } + + if (loom->rank_enabled && proc->rank < 0) { + err("missing rank in process %d", pid); + return -1; + } + + /* Check previous ranks if any */ + if (!loom->rank_enabled && proc->rank >= 0) { + loom->rank_enabled = 1; + loom->rank_min = INT_MAX; + + for (struct proc *p = loom->procs; p; p = p->hh.next) { + if (p->rank < 0) { + err("missing rank in process %d", p->pid); + return -1; + } + + if (p->rank < loom->rank_min) + loom->rank_min = p->rank; + } + } + + if (loom->rank_enabled && proc->rank < loom->rank_min) + loom->rank_min = proc->rank; + HASH_ADD_INT(loom->procs, pid, proc); loom->nprocs++; diff --git a/src/emu/loom.h b/src/emu/loom.h index 4381538..4a382d9 100644 --- a/src/emu/loom.h +++ b/src/emu/loom.h @@ -26,6 +26,7 @@ struct loom { size_t ncpus; size_t offset_ncpus; int rank_enabled; + int rank_min; int64_t clock_offset; diff --git a/src/emu/system.c b/src/emu/system.c index dc9abc3..596fdf4 100644 --- a/src/emu/system.c +++ b/src/emu/system.c @@ -84,11 +84,6 @@ create_proc(struct loom *loom, const char *tracedir, const char *relpath) return NULL; } - if (loom_add_proc(loom, proc) != 0) { - err("loom_add_proc failed"); - return NULL; - } - /* Build metadata path */ char mpath[PATH_MAX]; @@ -104,6 +99,11 @@ create_proc(struct loom *loom, const char *tracedir, const char *relpath) return NULL; } + if (loom_add_proc(loom, proc) != 0) { + err("loom_add_proc failed"); + return NULL; + } + return proc; } @@ -212,15 +212,32 @@ create_system(struct system *sys, struct trace *trace) } static int -cmp_loom(struct loom *a, struct loom *b) +cmp_loom_id(struct loom *a, struct loom *b) { return strcmp(a->id, b->id); } +static int +cmp_loom_rank(struct loom *a, struct loom *b) +{ + int id1 = a->rank_min; + int id2 = b->rank_min; + + if (id1 < id2) + return -1; + if (id1 > id2) + return +1; + else + return 0; +} + static void sort_lpt(struct system *sys) { - DL_SORT(sys->looms, cmp_loom); + if (sys->sort_by_rank) + DL_SORT(sys->looms, cmp_loom_rank); + else + DL_SORT(sys->looms, cmp_loom_id); for (struct loom *l = sys->looms; l; l = l->next) loom_sort(l); @@ -459,6 +476,29 @@ init_offsets(struct system *sys, struct trace *trace) return 0; } +static int +set_sort_criteria(struct system *sys) +{ + for (struct loom *l = sys->looms; l; l = l->next) { + if (l->rank_enabled) { + sys->sort_by_rank = 1; + break; + } + } + + if (!sys->sort_by_rank) + return 0; + + for (struct loom *l = sys->looms; l; l = l->next) { + if (!l->rank_enabled) { + err("missing rank for loom %s", l->id); + return -1; + } + } + + return 0; +} + int system_init(struct system *sys, struct emu_args *args, struct trace *trace) { @@ -471,6 +511,11 @@ system_init(struct system *sys, struct emu_args *args, struct trace *trace) return -1; } + if (set_sort_criteria(sys) != 0) { + err("set_sort_criteria failed"); + return -1; + } + /* Ensure they are sorted so they are easier to read */ sort_lpt(sys); diff --git a/src/emu/system.h b/src/emu/system.h index 0eaad7b..964078e 100644 --- a/src/emu/system.h +++ b/src/emu/system.h @@ -29,6 +29,8 @@ struct system { size_t ncpus; /* Including virtual cpus */ size_t nphycpus; + int sort_by_rank; + struct loom *looms; struct proc *procs; struct thread *threads; diff --git a/test/emu/ovni/CMakeLists.txt b/test/emu/ovni/CMakeLists.txt index 85929ab..fd161ad 100644 --- a/test/emu/ovni/CMakeLists.txt +++ b/test/emu/ovni/CMakeLists.txt @@ -17,3 +17,5 @@ test_emu(version-good.c) test_emu(version-bad.c SHOULD_FAIL REGEX "version mismatch") test_emu(clockgate.c MP SHOULD_FAIL REGEX "detected large clock gate") test_emu(no-cpus.c SHOULD_FAIL REGEX "loom .* has no physical CPUs") +test_emu(sort-cpus-by-loom.c MP) +test_emu(sort-cpus-by-rank.c MP) diff --git a/test/emu/ovni/sort-cpus-by-loom.c b/test/emu/ovni/sort-cpus-by-loom.c new file mode 100644 index 0000000..7fe4f3f --- /dev/null +++ b/test/emu/ovni/sort-cpus-by-loom.c @@ -0,0 +1,69 @@ +/* Copyright (c) 2023 Barcelona Supercomputing Center (BSC) + * SPDX-License-Identifier: GPL-3.0-or-later */ + +#include +#include +#include +#include "common.h" +#include "compat.h" +#include "instr.h" +#include "ovni.h" + +#define N 4 + +/* Ensures that in the CPU timeline, the order of the CPUs is given by the loom + * alphanumeric order, when no rank information is present. */ +int +main(void) +{ + int rank = atoi(getenv("OVNI_RANK")); + int nranks = atoi(getenv("OVNI_NRANKS")); + + int cpus[N]; + + for (int i = 0; i < N; i++) { + cpus[i] = rank * N + i; + } + + char loom[128]; + if (snprintf(loom, 128, "loom.%04d", nranks - rank) >= 128) + die("snprintf failed"); + + ovni_proc_init(1, loom, getpid()); + ovni_thread_init(get_tid()); + + for (int i = 0; i < N; i++) + ovni_add_cpu(i, cpus[i]); + + instr_thread_execute(-1, -1, 0); + + instr_end(); + + if (rank == 0) { + FILE *c = fopen("expected", "w"); + if (c == NULL) + die("fopen failed:"); + + /* The expected order should be increasing loom id but the CPUs + * should start from the end */ + for (int i = 0; i < nranks; i++) { + int k = nranks - 1 - i; + for (int j = k * N; j < (k + 1) * N; j++) { + fprintf(c, " CPU %d.%d\n", i, j); + } + } + + fclose(c); + + FILE *f = fopen("match.sh", "w"); + if (f == NULL) + die("fopen failed:"); + + fprintf(f, "grep ' CPU' ovni/cpu.row > found\n"); + fprintf(f, "diff -y expected found\n"); + + fclose(f); + } + + return 0; +} diff --git a/test/emu/ovni/sort-cpus-by-rank.c b/test/emu/ovni/sort-cpus-by-rank.c new file mode 100644 index 0000000..b919cb7 --- /dev/null +++ b/test/emu/ovni/sort-cpus-by-rank.c @@ -0,0 +1,68 @@ +/* Copyright (c) 2023 Barcelona Supercomputing Center (BSC) + * SPDX-License-Identifier: GPL-3.0-or-later */ + +#include +#include +#include +#include "common.h" +#include "compat.h" +#include "instr.h" +#include "ovni.h" + +#define N 4 + +/* Ensures that in the CPU trace, the order of the CPUs is given by the minimum + * rank of the processes of that loom, when the rank information is present. */ +int +main(void) +{ + int rank = atoi(getenv("OVNI_RANK")); + int nranks = atoi(getenv("OVNI_NRANKS")); + + int cpus[N]; + + for (int i = 0; i < N; i++) { + cpus[i] = rank * N + i; + } + + char loom[128]; + if (snprintf(loom, 128, "loom.%04d", nranks - rank) >= 128) + die("snprintf failed"); + + ovni_proc_init(1, loom, getpid()); + ovni_thread_init(get_tid()); + ovni_proc_set_rank(rank, nranks); + + for (int i = 0; i < N; i++) + ovni_add_cpu(i, cpus[i]); + + instr_thread_execute(-1, -1, 0); + + instr_end(); + + if (rank == 0) { + FILE *c = fopen("expected", "w"); + if (c == NULL) + die("fopen failed:"); + + /* The expected order should be increasing rank and CPUs */ + for (int i = 0; i < nranks; i++) { + for (int j = i * N; j < (i + 1) * N; j++) { + fprintf(c, " CPU %d.%d\n", i, j); + } + } + + fclose(c); + + FILE *f = fopen("match.sh", "w"); + if (f == NULL) + die("fopen failed:"); + + fprintf(f, "grep ' CPU' ovni/cpu.row > found\n"); + fprintf(f, "diff -y expected found\n"); + + fclose(f); + } + + return 0; +} diff --git a/test/unit/loom.c b/test/unit/loom.c index fdd621a..6283f90 100644 --- a/test/unit/loom.c +++ b/test/unit/loom.c @@ -67,6 +67,7 @@ test_duplicate_procs(struct loom *loom) struct proc proc; OK(loom_init_begin(loom, testloom)); OK(proc_init_begin(&proc, testproc)); + proc.metadata_loaded = 1; OK(loom_add_proc(loom, &proc)); ERR(loom_add_proc(loom, &proc));