Add OpenMP support for labels and taskID views

Co-authored-by: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>

.gitea/workflows/ci.yaml

@@ -0,0 +1,45 @@
+name: CI
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  build:local:
+    runs-on: native
+    steps:
+      - uses: https://gitea.com/ScMi1/checkout@v1.4
+      - run: nix build -L --no-link .#ovniPackages.local
+  
+  build:rt:
+    runs-on: native
+    steps:
+      - uses: https://gitea.com/ScMi1/checkout@v1.4
+      - run: nix build -L --no-link .#ovniPackages.rt
+
+  build:debug:
+    runs-on: native
+    steps:
+      - uses: https://gitea.com/ScMi1/checkout@v1.4
+      - run: nix build -L --no-link .#ovniPackages.debug
+  
+  build:asan:
+    runs-on: native
+    steps:
+      - uses: https://gitea.com/ScMi1/checkout@v1.4
+      - run: nix build -L --no-link .#ovniPackages.asan
+  
+  build:nompi:
+    runs-on: native
+    steps:
+      - uses: https://gitea.com/ScMi1/checkout@v1.4
+      - run: nix build -L --no-link .#ovniPackages.nompi
+  
+  build:compilers:
+    runs-on: native
+    steps:
+      - uses: https://gitea.com/ScMi1/checkout@v1.4
+      - run: nix build -L --no-link .#ovniPackages.compilers

.gitlab-ci.yml

@@ -14,44 +14,72 @@ build:debian-testing:
     paths:
       - install/
 
-build:rt-tests:
+build:local:
   stage: build
   tags:
     - nix
   script:
-    - nix build -L --no-link --tarball-ttl 0 --file nix/rt.nix bsc.ovni-rt
+    - nix build -L --no-link .#ovniPackages.local
+
+build:rt:
+  stage: build
+  tags:
+    - nix
+  script:
+    - nix build -L --no-link .#ovniPackages.rt
+
+build:debug:
+  stage: build
+  tags:
+    - nix
+  script:
+    - nix build -L --no-link .#ovniPackages.debug
 
 build:asan:
   stage: build
   tags:
     - nix
   script:
-    - nix build -L --no-link --tarball-ttl 0 --file nix/rt.nix bsc.ovni-asan
+    - nix build -L --no-link .#ovniPackages.asan
 
-build:no-mpi:
+build:ubsan:
   stage: build
   tags:
     - nix
   script:
-    - nix build -L --no-link --tarball-ttl 0 --file nix/rt.nix bsc.ovni-nompi
+    - nix build -L --no-link .#ovniPackages.ubsan
 
-build:old-gcc:
+build:nompi:
   stage: build
   tags:
     - nix
   script:
-    - nix build -L --no-link --tarball-ttl 0 --file nix/rt.nix bsc.oldOvnis
+    - nix build -L --no-link .#ovniPackages.nompi
 
-build:old-gcc-no-lto:
+build:compilers:
   stage: build
   tags:
     - nix
   script:
-    - nix build -L --no-link --tarball-ttl 0 --file nix/rt.nix bsc.oldOvnisNoLTO
+    - nix build -L --no-link .#ovniPackages.compilers
 
-build:old-gcc-release:
+build:armv7:
   stage: build
   tags:
     - nix
   script:
-    - nix build -L --no-link --tarball-ttl 0 --file nix/rt.nix bsc.oldOvnisRelease
+    - nix build -L --no-link .#ovniPackages.armv7
+
+build:aarch64:
+  stage: build
+  tags:
+    - nix
+  script:
+    - nix build -L --no-link .#ovniPackages.aarch64
+
+build:riscv64:
+  stage: build
+  tags:
+    - nix
+  script:
+    - nix build -L --no-link .#ovniPackages.riscv64

.readthedocs.yaml

@@ -0,0 +1,20 @@
+# .readthedocs.yaml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the version of Python and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+
+mkdocs:
+  configuration: mkdocs.yml
+  fail_on_warning: false
+
+python:
+  install:
+  - requirements: doc/requirements.txt

CHANGELOG.md

@@ -7,6 +7,175 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+
+- OpenMP model version increased to 1.2.0.
+
+### Added
+
+- Add support OpenMP label and task ID views.
+
+## [1.11.0] - 2024-11-08
+
+### Added
+
+- Introduce part model.
+- Support for `nosv_cond_wait`, `nosv_cond_signal` and `nosv_cond_broadcast` events VA{oOgGkK}.
+
+### Changed
+
+- Enable -Wconversion and -Wsign-conversion.
+- Update trace format to version 3.
+- The ovni.require metadata key is now mandatory.
+- Store process metadata in thread metadata.
+- nOS-V model version increased to 2.4.0.
+
+### Fixed
+
+- Fix bug in ovnisort when injecting events in a previously modified section.
+
+## [1.10.0] - 2024-07-26
+
+### Changed
+
+- nOS-V model version increased to 2.3.0.
+- Prevent accidental use of nOS-V traces without required events for the
+  breakdown model using the `nosv.can_breakdown` attribute.
+- Increase ovni model version to 1.1.0 for the mark events `OM*`.
+
+### Added
+
+- Add support for nOS-V progressing events VP{pra}.
+- Add breakdown model for nOS-V.
+- New API to manage stream metadata `ovni_attr_*()`.
+- New mark API `ovni_mark_*()` to emit user-defined events.
+
+### Fixed
+
+- Cross-compilation for ARM 32 bits.
+
+## [1.9.1] - 2024-05-10
+
+### Fixed
+
+- Fix breakdown model error that was preventing a zero value to be written in
+  the PRV trace.
+- Fix gcc 11.3.1 -Wstringop-overflow warning.
+
+## [1.9.0] - 2024-04-25
+
+### Added
+
+- Consistency check for events emitted when the kernel has removed the thread
+  from the CPU.
+- The nOS-V model version is bumped to 2.1.0.
+- Add support for `nosv_mutex_lock`, `nosv_mutex_trylock` and
+  `nosv_mutex_unlock` events VA{lLtTuU}.
+- Add support for the `nosv_barrier_event` event VA{bB}.
+
+### Fixed
+
+- Open Paraver windows at (100,100) instead of (0,0) to avoid them appearing
+  under the menu bar in Mac OS.
+
+## [1.8.0] - 2024-03-18
+
+### Added
+
+- Add new body model to support parallel tasks in nOS-V (used in taskfor).
+- Add the ability to restrict transitions in the task model states.
+- Add nOS-V support for parallel tasks reading the body id from the
+  event payload.
+- Keep a changelog of emulation model versions.
+
+### Changed
+
+- The task model now requires the previous task body to be paused before nesting
+  another one. A compatibility flag is still available to still allow the old
+  behavior.
+- Nanos6 model version increased to 1.1.0.
+- nOS-V model version increased to 2.0.0.
+
+## [1.7.0] - 2024-03-12
+
+### Added
+
+- Add OpenMP model (`P`) at version 1.1.0 (currently it only supports subsystems
+  and only works with the OpenMP-V runtime, on top of nOS-V).
+
+### Changed
+
+- Add support for `nosv_attach` and `nosv_detach` events VA{aAeE}.
+- Ignore old nOS-V attach events VH{aA}.
+- The nOS-V model version is bumped to 1.1.0.
+
+## [1.6.0] - 2024-02-14
+
+### Changed
+
+- All events now must be declared for each emulation model and given a
+  description including the payload arguments.
+
+### Added
+
+- Manual page for ovnidump(1).
+- Describe events in human readable format with ovnidump.
+- New `-x` option in ovnidump to show the payload in hexadecimal.
+
+## [1.5.1] - 2023-12-20
+
+### Fixed
+
+- Calling `ovni_thread_isready()` after `ovni_thread_free()` now returns 0.
+- Emitting events in a non-ready thread now aborts the program with a
+  message rather than causing a segfault.
+
+## [1.5.0] - 2023-12-15
+
+### Added
+
+- New function `ovni_thread_require()` to selectively enable emulation models
+  and check their version is compatible (if not used all models will be
+  enabled).
+- Support for per-thread metadata
+- Store the version of libovni in the metadata
+- Streams are marked as finished when `ovni_thread_free()` is called. A warning
+  is emitted in the emulator for those streams that are not finished properly.
+- List the emulation models and versions with `ovniemu -h`
+- New `-a` ovniemu option to enable all models
+
+### Changed
+
+- Updated process metadata to version 2 (traces generated with an older libovni
+  are not compatible with the emulator).
+- Emulation models now have a semantic version (X.Y.Z) instead of just a number.
+- Install ovniver with the runpath set.
+
+### Fixed
+
+- Close stream FD on `ovni_thread_free()`.
+
+## [1.4.1] - 2023-11-16
+
+### Changed
+
+- Fix emulation for level 2 or lower in nOS-V with inline tasks by
+  allowing duplicates in the subsystem channel.
+
+## [1.4.0] - 2023-11-08
+
+### Added
+
+- Add `OVNI_TRACEDIR` envar to change the trace directory (default is `ovni`).
+
+### Changed
+
+- Don't modify nOS-V subsystem state on task pause. The "Task: Running"
+  state is now renamed to "Task: In body" to reflect the change.
+- Use pkg-config to locate the nOS-V library and get the version. Use
+  `PKG_CONFIG_LIBDIR=/path/to/nosv/install/lib/pkgconfig` to use a custom
+  installation path.
+
 ## [1.3.0] - 2023-09-07
 
 ### Added
@@ -123,10 +292,21 @@ are used along with some other changes.
 
 - First ovni release.
 
-[unreleased]: https://pm.bsc.es/gitlab/rarias/ovni/-/commits/master
-[1.3.0]: https://pm.bsc.es/gitlab/rarias/ovni/-/tags/1.3.0
-[1.2.2]: https://pm.bsc.es/gitlab/rarias/ovni/-/tags/1.2.2
-[1.2.1]: https://pm.bsc.es/gitlab/rarias/ovni/-/tags/1.2.1
-[1.2.0]: https://pm.bsc.es/gitlab/rarias/ovni/-/tags/1.2.0
-[1.1.0]: https://pm.bsc.es/gitlab/rarias/ovni/-/tags/1.1.0
-[1.0.0]: https://pm.bsc.es/gitlab/rarias/ovni/-/tags/1.0.0
+[unreleased]: https://jungle.bsc.es/git/rarias/ovni
+[1.11.0]: https://github.com/rodarima/ovni/releases/tag/1.11.0
+[1.10.0]: https://github.com/rodarima/ovni/releases/tag/1.10.0
+[1.9.1]: https://github.com/rodarima/ovni/releases/tag/1.9.1
+[1.9.0]: https://github.com/rodarima/ovni/releases/tag/1.9.0
+[1.8.0]: https://github.com/rodarima/ovni/releases/tag/1.8.0
+[1.7.0]: https://github.com/rodarima/ovni/releases/tag/1.7.0
+[1.6.0]: https://github.com/rodarima/ovni/releases/tag/1.6.0
+[1.5.1]: https://github.com/rodarima/ovni/releases/tag/1.5.1
+[1.5.0]: https://github.com/rodarima/ovni/releases/tag/1.5.0
+[1.4.1]: https://github.com/rodarima/ovni/releases/tag/1.4.1
+[1.4.0]: https://github.com/rodarima/ovni/releases/tag/1.4.0
+[1.3.0]: https://github.com/rodarima/ovni/releases/tag/1.3.0
+[1.2.2]: https://github.com/rodarima/ovni/releases/tag/1.2.2
+[1.2.1]: https://github.com/rodarima/ovni/releases/tag/1.2.1
+[1.2.0]: https://github.com/rodarima/ovni/releases/tag/1.2.0
+[1.1.0]: https://github.com/rodarima/ovni/releases/tag/1.1.0
+[1.0.0]: https://github.com/rodarima/ovni/releases/tag/1.0.0

CMakeLists.txt

@@ -1,15 +1,15 @@
-# Copyright (c) 2021-2023 Barcelona Supercomputing Center (BSC)
+# Copyright (c) 2021-2024 Barcelona Supercomputing Center (BSC)
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 cmake_minimum_required(VERSION 3.20)
 
-project(OVNI LANGUAGES C VERSION 1.3.0)
+project(OVNI LANGUAGES C VERSION 1.11.0)
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/")
 
 add_compile_options(-Wall -Wextra -Wformat
   -Wmissing-prototypes -Wstrict-prototypes
-  #-Wconversion -Wsign-conversion
+  -Wconversion -Wsign-conversion
   -Wold-style-definition -pedantic
   -Werror
 )
@@ -67,7 +67,9 @@ set(CMAKE_C_FLAGS_ASAN "${CMAKE_C_FLAGS_DEBUG} \
   -fno-omit-frame-pointer"
   CACHE STRING "Flags used by the C compiler during AddressSanitizer builds." FORCE)
 
-set(CMAKE_C_FLAGS_UBSAN "${CMAKE_C_FLAGS_DEBUG} -fsanitize=undefined"
+set(CMAKE_C_FLAGS_UBSAN "${CMAKE_C_FLAGS_DEBUG} \
+  -fsanitize=undefined \
+  -fno-sanitize-recover=all"
   CACHE STRING "Flags used by the C compiler during UndefinedBehaviorSanitizer builds." FORCE)
 
 find_program(IWYU NAMES include-what-you-use iwyu)
@@ -103,3 +105,6 @@ if(BUILD_TESTING)
 endif()
 
 install(DIRECTORY cfg/ DESTINATION "${OVNI_CONFIG_RELDIR}")
+
+include(FeatureSummary)
+feature_summary(WHAT ALL)

cfg/cpu/kernel/context-switch.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name CPU: Context switches of the RUNNING thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled true

cfg/cpu/mpi/function.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name CPU: MPI function of the RUNNING thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled true

cfg/cpu/nanos6/breakdown.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name CPU: Nanos6 Runtime/Idle/Task breakdown
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled false

cfg/cpu/nanos6/idle.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name CPU: Nanos6 idle state of the RUNNING thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled false

cfg/cpu/nanos6/mpi-rank.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name CPU: Nanos6 task MPI rank of the RUNNING thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled false

cfg/cpu/nanos6/subsystem.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name CPU: Nanos6 subsystem of the RUNNING thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled false

cfg/cpu/nanos6/task-id.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name CPU: Nanos6 task ID of the RUNNING thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled false

cfg/cpu/nanos6/task-type.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name CPU: Nanos6 task type of the RUNNING thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled false

cfg/cpu/nanos6/thread-type.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name CPU: Nanos6 thread type of the RUNNING thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled false

cfg/cpu/nodes/subsystem.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name CPU: NODES subsystem of the RUNNING thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled true

cfg/cpu/nosv/body-id.cfg

@@ -0,0 +1,42 @@
+#ParaverCFG
+ConfigFile.Version: 3.4
+ConfigFile.NumWindows: 1
+
+
+################################################################################
+< NEW DISPLAYING WINDOW CPU: nOS-V task body ID of the RUNNING thread >
+################################################################################
+window_name CPU: nOS-V task body ID of the RUNNING thread
+window_type single
+window_id 1
+window_position_x 100
+window_position_y 100
+window_width 600
+window_height 150
+window_comm_lines_enabled true
+window_flags_enabled false
+window_noncolor_mode true
+window_color_mode window_in_null_gradient_mode
+window_logical_filtered true
+window_physical_filtered false
+window_comm_fromto true
+window_comm_tagsize true
+window_comm_typeval true
+window_units Microseconds
+window_maximum_y 1000.0
+window_minimum_y 1.0
+window_compute_y_max true
+window_level thread
+window_scale_relative 1.000000000000
+window_end_time_relative 1.000000000000
+window_object appl { 1, { All } }
+window_begin_time_relative 0.000000000000
+window_open true
+window_drawmode draw_randnotzero
+window_drawmode_rows draw_randnotzero
+window_pixel_size 1
+window_labels_to_draw 1
+window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } }
+window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } }
+window_filter_module evt_type 1 15
+window_filter_module evt_type_label 1 "CPU: nOS-V task body ID of the RUNNING thread"

cfg/cpu/nosv/breakdown.cfg

@@ -0,0 +1,44 @@
+#ParaverCFG
+ConfigFile.Version: 3.4
+ConfigFile.NumWindows: 1
+
+
+################################################################################
+< NEW DISPLAYING WINDOW CPU: nOS-V Runtime/Idle/Task breakdown >
+################################################################################
+window_name CPU: nOS-V Runtime/Idle/Task breakdown
+window_type single
+window_id 1
+window_position_x 0
+window_position_y 0
+window_width 600
+window_height 150
+window_comm_lines_enabled false
+window_flags_enabled false
+window_noncolor_mode true
+window_custom_color_enabled true
+window_custom_color_palette {6.000000000000:94,0,0},{7.000000000000:153,114,0},{9.000000000000:124,213,228},{10.000000000000:242,239,141},{11.000000000000:0,70,0},{19.000000000000:195,96,151},{20.000000000000:255,162,255},{21.000000000000:203,255,3},{22.000000000000:7,255,12},{23.000000000000:21,224,189},{24.000000000000:255,103,0},{25.000000000000:0,99,162},{26.000000000000:110,77,252},{100.000000000000:0,100,0},{101.000000000000:60,60,60},{102.000000000000:150,150,0}
+window_logical_filtered true
+window_physical_filtered false
+window_comm_fromto true
+window_comm_tagsize true
+window_comm_typeval true
+window_units Microseconds
+window_maximum_y 1000.0
+window_minimum_y 1.0
+window_compute_y_max true
+window_level thread
+window_scale_relative 1.000000000000
+window_end_time_relative 1.000000000000
+window_object appl { 1, { All } }
+window_begin_time_relative 0.000000000000
+window_open true
+window_drawmode draw_randnotzero
+window_drawmode_rows draw_randnotzero
+window_pixel_size 1
+window_labels_to_draw 1
+window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } }
+window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } }
+window_filter_module evt_type 1 17
+window_filter_module evt_type_label 1 "CPU: nOS-V Runtime/Idle/Task breakdown"
+

cfg/cpu/nosv/idle.cfg

@@ -0,0 +1,44 @@
+#ParaverCFG
+ConfigFile.Version: 3.4
+ConfigFile.NumWindows: 1
+
+
+################################################################################
+< NEW DISPLAYING WINDOW CPU: nOS-V idle state of the RUNNING thread >
+################################################################################
+window_name CPU: nOS-V idle state of the RUNNING thread
+window_type single
+window_id 1
+window_position_x 0
+window_position_y 0
+window_width 600
+window_height 150
+window_comm_lines_enabled false
+window_flags_enabled false
+window_noncolor_mode true
+window_custom_color_enabled true
+window_custom_color_palette {100.000000000000:0,100,0},{101.000000000000:60,60,60},{102.000000000000:150,150,0}
+window_logical_filtered true
+window_physical_filtered false
+window_comm_fromto true
+window_comm_tagsize true
+window_comm_typeval true
+window_units Microseconds
+window_maximum_y 1000.0
+window_minimum_y 1.0
+window_compute_y_max true
+window_level thread
+window_scale_relative 1.000000000000
+window_end_time_relative 1.000000000000
+window_object appl { 1, { All } }
+window_begin_time_relative 0.000000000000
+window_open true
+window_drawmode draw_randnotzero
+window_drawmode_rows draw_randnotzero
+window_pixel_size 1
+window_labels_to_draw 1
+window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } }
+window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } }
+window_filter_module evt_type 1 16
+window_filter_module evt_type_label 1 "CPU: nOS-V idle state of the RUNNING thread"
+

cfg/cpu/nosv/subsystem.cfg

@@ -9,15 +9,15 @@ ConfigFile.NumWindows: 1
 window_name CPU: nOS-V subsystem of the RUNNING thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled true
 window_flags_enabled false
 window_noncolor_mode true
 window_custom_color_enabled true
-window_custom_color_palette {6.000000000000:94,0,0},{7.000000000000:153,114,0},{9.000000000000:124,213,228},{10.000000000000:242,239,141},{11.000000000000:0,70,0}
+window_custom_color_palette {6.000000000000:94,0,0},{7.000000000000:153,114,0},{9.000000000000:124,213,228},{10.000000000000:242,239,141},{11.000000000000:0,70,0},{19.000000000000:195,96,151},{20.000000000000:255,162,255},{21.000000000000:203,255,3},{22.000000000000:7,255,12},{23.000000000000:21,165,118},{24.000000000000:255,103,0},{25.000000000000:200,30,5},{26.000000000000:255,10,200},{27.000000000000:98,133,80},{28.000000000000:0,99,162},{29.000000000000:110,77,252}
 window_logical_filtered true
 window_physical_filtered false
 window_comm_fromto true

cfg/cpu/nosv/task-appid.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name CPU: nOS-V task AppID of the RUNNING thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled true

cfg/cpu/nosv/task-id.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name CPU: nOS-V TaskID of the RUNNING thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled true

cfg/cpu/nosv/task-rank.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name CPU: nOS-V task rank+1 of the RUNNING thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled true

cfg/cpu/nosv/task-type.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name CPU: nOS-V task type of the RUNNING thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled true

cfg/cpu/openmp/label.cfg

@@ -0,0 +1,41 @@
+#ParaverCFG
+ConfigFile.Version: 3.4
+ConfigFile.NumWindows: 1
+
+
+################################################################################
+< NEW DISPLAYING WINDOW CPU: OpenMP label of the RUNNING thread >
+################################################################################
+window_name CPU: OpenMP label of the RUNNING thread
+window_type single
+window_id 1
+window_position_x 100
+window_position_y 100
+window_width 600
+window_height 150
+window_comm_lines_enabled true
+window_flags_enabled false
+window_noncolor_mode true
+window_logical_filtered true
+window_physical_filtered false
+window_comm_fromto true
+window_comm_tagsize true
+window_comm_typeval true
+window_units Microseconds
+window_maximum_y 1000.0
+window_minimum_y 1.0
+window_compute_y_max true
+window_level thread
+window_scale_relative 1.000000000000
+window_end_time_relative 1.000000000000
+window_object appl { 1, { All } }
+window_begin_time_relative 0.000000000000
+window_open true
+window_drawmode draw_randnotzero
+window_drawmode_rows draw_randnotzero
+window_pixel_size 1
+window_labels_to_draw 1
+window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } }
+window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } }
+window_filter_module evt_type 1 51
+window_filter_module evt_type_label 1 "CPU: OpenMP label of the RUNNING thread"

cfg/cpu/openmp/subsystem.cfg

@@ -0,0 +1,44 @@
+#ParaverCFG
+ConfigFile.Version: 3.4
+ConfigFile.NumWindows: 1
+
+
+################################################################################
+< NEW DISPLAYING WINDOW CPU: OpenMP subsystem of the RUNNING thread >
+################################################################################
+window_name CPU: OpenMP subsystem of the RUNNING thread
+window_type single
+window_id 1
+window_position_x 100
+window_position_y 100
+window_width 600
+window_height 150
+window_comm_lines_enabled true
+window_flags_enabled false
+window_noncolor_mode true
+window_custom_color_enabled true
+window_custom_color_palette {1.000000000000:255,177,245},{2.000000000000:255,86,239},{3.000000000000:122,44,22},{5.000000000000:239,188,0},{6.000000000000:160,89,0},{8.000000000000:0,255,73},{10.000000000000:86,209,43},{11.000000000000:203,208,93},{12.000000000000:0,176,169},{13.000000000000:190,82,201},{14.000000000000:124,114,183},{15.000000000000:157,231,255},{16.000000000000:199,194,0},{17.000000000000:96,0,200},{18.000000000000:255,255,124},{19.000000000000:35,152,0},{21.000000000000:255,251,174},{22.000000000000:232,0,0},{23.000000000000:210,66,40},{26.000000000000:101,101,99},{27.000000000000:200,0,255},{28.000000000000:0,203,249},{30.000000000000:255,219,0},{31.000000000000:48,103,107},{34.000000000000:194,105,126}
+window_logical_filtered true
+window_physical_filtered false
+window_comm_fromto true
+window_comm_tagsize true
+window_comm_typeval true
+window_units Microseconds
+window_maximum_y 1000.0
+window_minimum_y 1.0
+window_compute_y_max true
+window_level thread
+window_scale_relative 1.000000000000
+window_end_time_relative 1.000000000000
+window_object appl { 1, { All } }
+window_begin_time_relative 0.000000000000
+window_open true
+window_drawmode draw_randnotzero
+window_drawmode_rows draw_randnotzero
+window_pixel_size 1
+window_labels_to_draw 1
+window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } }
+window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } }
+window_filter_module evt_type 1 50
+window_filter_module evt_type_label 1 "CPU: OpenMP subsystem of the RUNNING thread"
+

cfg/cpu/openmp/task-id.cfg

@@ -0,0 +1,41 @@
+#ParaverCFG
+ConfigFile.Version: 3.4
+ConfigFile.NumWindows: 1
+
+
+################################################################################
+< NEW DISPLAYING WINDOW CPU: OpenMP task id of the RUNNING thread >
+################################################################################
+window_name CPU: OpenMP task id of the RUNNING thread
+window_type single
+window_id 1
+window_position_x 100
+window_position_y 100
+window_width 600
+window_height 150
+window_comm_lines_enabled true
+window_flags_enabled false
+window_noncolor_mode true
+window_logical_filtered true
+window_physical_filtered false
+window_comm_fromto true
+window_comm_tagsize true
+window_comm_typeval true
+window_units Microseconds
+window_maximum_y 1000.0
+window_minimum_y 1.0
+window_compute_y_max true
+window_level thread
+window_scale_relative 1.000000000000
+window_end_time_relative 1.000000000000
+window_object appl { 1, { All } }
+window_begin_time_relative 0.000000000000
+window_open true
+window_drawmode draw_randnotzero
+window_drawmode_rows draw_randnotzero
+window_pixel_size 1
+window_labels_to_draw 1
+window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } }
+window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } }
+window_filter_module evt_type 1 52
+window_filter_module evt_type_label 1 "CPU: OpenMP task id of the RUNNING thread"

cfg/cpu/ovni/flushing.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name CPU: Flushing state of the RUNNING thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled true

cfg/cpu/ovni/mark.cfg

@@ -0,0 +1,41 @@
+#ParaverCFG
+ConfigFile.Version: 3.4
+ConfigFile.NumWindows: 1
+
+
+################################################################################
+< NEW DISPLAYING WINDOW CPU: Mark value of the RUNNING thread >
+################################################################################
+window_name CPU: Mark value of the RUNNING thread
+window_type single
+window_id 1
+window_position_x 100
+window_position_y 100
+window_width 600
+window_height 150
+window_comm_lines_enabled true
+window_flags_enabled false
+window_noncolor_mode true
+window_logical_filtered true
+window_physical_filtered false
+window_comm_fromto true
+window_comm_tagsize true
+window_comm_typeval true
+window_units Microseconds
+window_maximum_y 1000.0
+window_minimum_y 1.0
+window_compute_y_max true
+window_level thread
+window_scale_relative 1.000000000000
+window_end_time_relative 1.000000000000
+window_object appl { 1, { All } }
+window_begin_time_relative 0.000000000000
+window_open true
+window_drawmode draw_randnotzero
+window_drawmode_rows draw_randnotzero
+window_pixel_size 1
+window_labels_to_draw 1
+window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } }
+window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } }
+window_filter_module evt_type 1 100
+window_filter_module evt_type_label 1 "CPU: Mark value of the RUNNING thread"

cfg/cpu/ovni/number-running-threads.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name CPU: Number of RUNNING threads
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled true

cfg/cpu/ovni/pid.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name CPU: PID of the RUNNING thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled true

cfg/cpu/ovni/tid.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name CPU: TID of the RUNNING thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled true

cfg/cpu/tampi/subsystem.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name CPU: TAMPI subsystem of the RUNNING thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled true

cfg/thread/kernel/context-switch.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name Thread: Context switches of the CURRENT thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled true

cfg/thread/mpi/function.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name Thread: MPI function of the RUNNING thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled true

cfg/thread/nanos6/idle.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name Thread: Nanos6 idle state
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled false

cfg/thread/nanos6/mpi-rank.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name Thread: Nanos6 task MPI rank of the RUNNING thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled false

cfg/thread/nanos6/subsystem.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name Thread: Nanos6 subsystem of the ACTIVE thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled false

cfg/thread/nanos6/task-id.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name Thread: Nanos6 task ID of the RUNNING thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled false

cfg/thread/nanos6/task-type.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name Thread: Nanos6 task type of the RUNNING thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled false

cfg/thread/nanos6/thread-type.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name Thread: Nanos6 thread type
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled false

cfg/thread/nodes/subsystem.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name Thread: NODES subsystem of the ACTIVE thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled true

cfg/thread/nosv/body-id.cfg

@@ -0,0 +1,42 @@
+#ParaverCFG
+ConfigFile.Version: 3.4
+ConfigFile.NumWindows: 1
+
+
+################################################################################
+< NEW DISPLAYING WINDOW Thread: nOS-V task body ID of the RUNNING thread >
+################################################################################
+window_name Thread: nOS-V task body ID of the RUNNING thread
+window_type single
+window_id 1
+window_position_x 100
+window_position_y 100
+window_width 600
+window_height 150
+window_comm_lines_enabled true
+window_flags_enabled false
+window_noncolor_mode true
+window_color_mode window_in_null_gradient_mode
+window_logical_filtered true
+window_physical_filtered false
+window_comm_fromto true
+window_comm_tagsize true
+window_comm_typeval true
+window_units Microseconds
+window_maximum_y 1000.0
+window_minimum_y 1.0
+window_compute_y_max true
+window_level thread
+window_scale_relative 1.000000000000
+window_end_time_relative 1.000000000000
+window_object appl { 1, { All } }
+window_begin_time_relative 0.000000000000
+window_open true
+window_drawmode draw_randnotzero
+window_drawmode_rows draw_randnotzero
+window_pixel_size 1
+window_labels_to_draw 1
+window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } }
+window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } }
+window_filter_module evt_type 1 15
+window_filter_module evt_type_label 1 "Thread: nOS-V task body ID of the RUNNING thread"

cfg/thread/nosv/subsystem.cfg

@@ -9,15 +9,15 @@ ConfigFile.NumWindows: 1
 window_name Thread: nOS-V subsystem of the ACTIVE thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled true
 window_flags_enabled false
 window_noncolor_mode true
 window_custom_color_enabled true
-window_custom_color_palette {6.000000000000:94,0,0},{7.000000000000:153,114,0},{9.000000000000:124,213,228},{10.000000000000:242,239,141},{11.000000000000:0,70,0}
+window_custom_color_palette {6.000000000000:94,0,0},{7.000000000000:153,114,0},{9.000000000000:124,213,228},{10.000000000000:242,239,141},{11.000000000000:0,70,0},{19.000000000000:195,96,151},{20.000000000000:255,162,255},{21.000000000000:203,255,3},{22.000000000000:7,255,12},{23.000000000000:21,165,118},{24.000000000000:255,103,0},{25.000000000000:200,30,5},{26.000000000000:255,10,200},{27.000000000000:98,133,80},{28.000000000000:0,99,162},{29.000000000000:110,77,252}
 window_logical_filtered true
 window_physical_filtered false
 window_comm_fromto true

cfg/thread/nosv/task-appid.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name Thread: nOS-V task AppID of the RUNNING thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled true

cfg/thread/nosv/task-id.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name Thread: nOS-V TaskID of the RUNNING thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled true

cfg/thread/nosv/task-rank.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name Thread: nOS-V task rank+1 of the RUNNING thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled true

cfg/thread/nosv/task-type.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name Thread: nOS-V task type of the RUNNING thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled true

cfg/thread/openmp/label.cfg

@@ -0,0 +1,41 @@
+#ParaverCFG
+ConfigFile.Version: 3.4
+ConfigFile.NumWindows: 1
+
+
+################################################################################
+< NEW DISPLAYING WINDOW Thread: OpenMP label of the ACTIVE thread >
+################################################################################
+window_name Thread: OpenMP label of the ACTIVE thread
+window_type single
+window_id 1
+window_position_x 100
+window_position_y 100
+window_width 600
+window_height 150
+window_comm_lines_enabled true
+window_flags_enabled false
+window_noncolor_mode true
+window_logical_filtered true
+window_physical_filtered false
+window_comm_fromto true
+window_comm_tagsize true
+window_comm_typeval true
+window_units Microseconds
+window_maximum_y 1000.0
+window_minimum_y 1.0
+window_compute_y_max true
+window_level thread
+window_scale_relative 1.000000000000
+window_end_time_relative 1.000000000000
+window_object appl { 1, { All } }
+window_begin_time_relative 0.000000000000
+window_open true
+window_drawmode draw_randnotzero
+window_drawmode_rows draw_randnotzero
+window_pixel_size 1
+window_labels_to_draw 1
+window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } }
+window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } }
+window_filter_module evt_type 1 51
+window_filter_module evt_type_label 1 "Thread: OpenMP label of the ACTIVE thread"

cfg/thread/openmp/subsystem.cfg

@@ -0,0 +1,44 @@
+#ParaverCFG
+ConfigFile.Version: 3.4
+ConfigFile.NumWindows: 1
+
+
+################################################################################
+< NEW DISPLAYING WINDOW Thread: OpenMP subsystem of the ACTIVE thread >
+################################################################################
+window_name Thread: OpenMP subsystem of the ACTIVE thread
+window_type single
+window_id 1
+window_position_x 100
+window_position_y 100
+window_width 600
+window_height 150
+window_comm_lines_enabled true
+window_flags_enabled false
+window_noncolor_mode true
+window_custom_color_enabled true
+window_custom_color_palette {1.000000000000:255,177,245},{2.000000000000:255,86,239},{3.000000000000:122,44,22},{5.000000000000:239,188,0},{6.000000000000:160,89,0},{8.000000000000:0,255,73},{10.000000000000:86,209,43},{11.000000000000:203,208,93},{12.000000000000:0,176,169},{13.000000000000:190,82,201},{14.000000000000:124,114,183},{15.000000000000:157,231,255},{16.000000000000:199,194,0},{17.000000000000:96,0,200},{18.000000000000:255,255,124},{19.000000000000:35,152,0},{21.000000000000:255,251,174},{22.000000000000:232,0,0},{23.000000000000:210,66,40},{26.000000000000:101,101,99},{27.000000000000:200,0,255},{28.000000000000:0,203,249},{30.000000000000:255,219,0},{31.000000000000:48,103,107},{34.000000000000:194,105,126}
+window_logical_filtered true
+window_physical_filtered false
+window_comm_fromto true
+window_comm_tagsize true
+window_comm_typeval true
+window_units Microseconds
+window_maximum_y 1000.0
+window_minimum_y 1.0
+window_compute_y_max true
+window_level thread
+window_scale_relative 1.000000000000
+window_end_time_relative 1.000000000000
+window_object appl { 1, { All } }
+window_begin_time_relative 0.000000000000
+window_open true
+window_drawmode draw_randnotzero
+window_drawmode_rows draw_randnotzero
+window_pixel_size 1
+window_labels_to_draw 1
+window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } }
+window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } }
+window_filter_module evt_type 1 50
+window_filter_module evt_type_label 1 "Thread: OpenMP subsystem of the ACTIVE thread"
+

cfg/thread/openmp/task-id.cfg

@@ -0,0 +1,41 @@
+#ParaverCFG
+ConfigFile.Version: 3.4
+ConfigFile.NumWindows: 1
+
+
+################################################################################
+< NEW DISPLAYING WINDOW Thread: OpenMP task id of the ACTIVE thread >
+################################################################################
+window_name Thread: OpenMP task id of the ACTIVE thread
+window_type single
+window_id 1
+window_position_x 100
+window_position_y 100
+window_width 600
+window_height 150
+window_comm_lines_enabled true
+window_flags_enabled false
+window_noncolor_mode true
+window_logical_filtered true
+window_physical_filtered false
+window_comm_fromto true
+window_comm_tagsize true
+window_comm_typeval true
+window_units Microseconds
+window_maximum_y 1000.0
+window_minimum_y 1.0
+window_compute_y_max true
+window_level thread
+window_scale_relative 1.000000000000
+window_end_time_relative 1.000000000000
+window_object appl { 1, { All } }
+window_begin_time_relative 0.000000000000
+window_open true
+window_drawmode draw_randnotzero
+window_drawmode_rows draw_randnotzero
+window_pixel_size 1
+window_labels_to_draw 1
+window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } }
+window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } }
+window_filter_module evt_type 1 52
+window_filter_module evt_type_label 1 "Thread: OpenMP task id of the ACTIVE thread"

cfg/thread/ovni/affinity.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name Thread: CPU affinity of the CURRENT thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled true

cfg/thread/ovni/flushing.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name Thread: Flushing state of the CURRENT thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled true

cfg/thread/ovni/mark.cfg

@@ -0,0 +1,41 @@
+#ParaverCFG
+ConfigFile.Version: 3.4
+ConfigFile.NumWindows: 1
+
+
+################################################################################
+< NEW DISPLAYING WINDOW Thread: Mark value of the ACTIVE thread >
+################################################################################
+window_name Thread: Mark value of the ACTIVE thread
+window_type single
+window_id 1
+window_position_x 100
+window_position_y 100
+window_width 600
+window_height 150
+window_comm_lines_enabled true
+window_flags_enabled false
+window_noncolor_mode true
+window_logical_filtered true
+window_physical_filtered false
+window_comm_fromto true
+window_comm_tagsize true
+window_comm_typeval true
+window_units Microseconds
+window_maximum_y 1000.0
+window_minimum_y 1.0
+window_compute_y_max true
+window_level thread
+window_scale_relative 1.000000000000
+window_end_time_relative 1.000000000000
+window_object appl { 1, { All } }
+window_begin_time_relative 0.000000000000
+window_open true
+window_drawmode draw_randnotzero
+window_drawmode_rows draw_randnotzero
+window_pixel_size 1
+window_labels_to_draw 1
+window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } }
+window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } }
+window_filter_module evt_type 1 100
+window_filter_module evt_type_label 1 "Thread: Mark value of the ACTIVE thread"

cfg/thread/ovni/pid.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name CPU: PID of the RUNNING thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled true

cfg/thread/ovni/state.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name Thread: State of the CURRENT thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled true

cfg/thread/ovni/tid.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name Thread: TID of the ACTIVE thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled true

cfg/thread/tampi/subsystem.cfg

@@ -9,8 +9,8 @@ ConfigFile.NumWindows: 1
 window_name Thread: TAMPI subsystem of the ACTIVE thread
 window_type single
 window_id 1
-window_position_x 0
-window_position_y 0
+window_position_x 100
+window_position_y 100
 window_width 600
 window_height 150
 window_comm_lines_enabled true

cmake/CheckOmpSs2Compiler.cmake

@@ -1,3 +1,6 @@
+# Copyright (c) 2022-2023 Barcelona Supercomputing Center (BSC)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
 include(CheckCCompilerFlag)
 
 set(CMAKE_REQUIRED_LINK_OPTIONS "-fompss-2")

cmake/CheckPerfParanoid.cmake

@@ -0,0 +1,20 @@
+# Copyright (c) 2024 Barcelona Supercomputing Center (BSC)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+if(EXISTS "/proc/sys/kernel/perf_event_paranoid")
+  file(READ "/proc/sys/kernel/perf_event_paranoid" paranoid_raw)
+
+  string(REPLACE "\n" "" paranoid_value "${paranoid_raw}")
+  message(STATUS "Value of /proc/sys/kernel/perf_event_paranoid is ${paranoid_value}")
+
+  if(paranoid_value LESS_EQUAL 1)
+    message(STATUS "Value of perf_event_paranoid suitable for Kernel tests")
+    set(PERF_PARANOID_KERNEL ON)
+  else()
+    message(STATUS "Value of perf_event_paranoid NOT suitable for Kernel tests")
+    set(PERF_PARANOID_KERNEL OFF)
+  endif()
+else()
+  message(STATUS "Missing /proc/sys/kernel/perf_event_paranoid")
+  set(PERF_PARANOID_KERNEL OFF)
+endif()

cmake/FindNanos6.cmake

@@ -1,3 +1,6 @@
+# Copyright (c) 2022-2023 Barcelona Supercomputing Center (BSC)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
 include(GNUInstallDirs)
 
 find_library(NANOS6_LIBRARY NAMES nanos6)

cmake/FindNodes.cmake

@@ -1,3 +1,6 @@
+# Copyright (c) 2022-2023 Barcelona Supercomputing Center (BSC)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
 include(GNUInstallDirs)
 
 if(DEFINED ENV{NODES_HOME})

cmake/FindNosv.cmake

@@ -0,0 +1,19 @@
+# Copyright (c) 2023 Barcelona Supercomputing Center (BSC)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+find_package(PkgConfig)
+
+if(NOT PKG_CONFIG_FOUND)
+  message(STATUS "pkg-config not found, required to locate nOSV-V")
+  return()
+endif()
+
+# Use PKG_CONFIG_LIBDIR=/path/to/nosv/install/lib/pkgconfig to use a custom
+# installation.
+pkg_search_module(NOSV IMPORTED_TARGET nos-v)
+
+if(NOT NOSV_FOUND)
+  message(STATUS "nOS-V not found")
+else()
+  message(STATUS "Found nOS-V ${NOSV_VERSION} at ${NOSV_LINK_LIBRARIES}")
+endif()

doc/CHANGELOG.md

@@ -0,0 +1 @@
+../CHANGELOG.md

doc/dev/channels.md

@@ -65,5 +65,5 @@ to write the duplicated value with no error.
 A unique function can be set to each channel which will be called once a channel
 becomes dirty with `chan_set_dirty_cb()`. This callback will be called before
 `chan_set()`, `chan_push()` or `chan_pop()` returns. The [patch
-bay](../patchbay) uses this callback to detect when a channel is modified an run
+bay](patchbay.md) uses this callback to detect when a channel is modified an run
 other callbacks.

doc/dev/events.md

@@ -0,0 +1,101 @@
+# Emulator events
+
+The events processed by the emulator are described in each model.
+Unrecognized events will cause a panic and stop the emulator in most
+cases.
+
+The events may have additional arguments in the payload, which are also
+described. To this end, a simple language was created to specify the
+format of each event in a concise declaration.
+
+Additionally, a printf-like string is declared for each event, so they
+can be explained in plain English. The values of the arguments are also
+substituted in the description of the event following a extended printf
+format described below.
+
+## Event format
+
+The events are defined by a small language that defines the MCV of an event, if
+is a jumbo and the arguments in the payload (if any).
+
+The grammar of this language is as follows in [ABNF][abnf].
+
+[abnf]: https://en.wikipedia.org/wiki/Augmented_Backus%E2%80%93Naur_form
+
+```ABNF
+event-definition = event-mcv event-type [ "(" argument-list ")" ]
+event-mcv = VCHAR VCHAR VCHAR
+event-type = [ '+' ]
+argument-list = argument-type " " argument-name [ ", " argument-list ]
+argument-name = 1*(CHAR / DIGIT)
+argument-type = type-signed | type-unsigned | type-string
+type-signed = "i8" | "i16" | "i32" | "i64"
+type-unsigned = "u8" | "u16" | "u32" | "u64"
+type-string = "str"
+```
+
+The `event-type` defines the type of event. Using the symbol `+` defines
+the event as jumbo. Otherwise, if not given it is considered a normal
+event.
+
+Here are some examples:
+
+- `OHp`: A normal event with `OHp` MCV and no payload.
+
+- `OAs(i32 cpu)`: A normal event with `OAs` MCV that has the cpu stored in
+  the payload as a 32 bits signed integer.
+
+- `OHC(i32 cpu, u64 tag)`: A normal event with `OHC` MCV that has two
+  arguments in the payload: the cpu stored as a 32 bit signed integer,
+  and a tag stored as a 64 bit unsigned integer.
+
+- `VYc+(u32 typeid, str label)`: A jumbo event with `VTc` MCV that has in the
+  jumbo payload a 32 bits unsigned integer for the typeid followed by the label
+  null-terminated string of variable size.
+
+## Event description
+
+To describe the meaning of each event, a description follows the event
+declaration. This description accepts printf-like format specifiers that
+are replaced with the value of the argument they refer to.
+
+The formats are specified as follows:
+
+```ABNF
+format-specifier = "%" [ printf-format] "{" argument-name "}"
+argument-name = 1*(CHAR / DIGIT)
+```
+
+Where the optional `printf-format` is any string accepted by the format
+specifier of `printf()`, as defined in the manual `printf(3)`. If the
+`printf-format` is not given, the default format for the argument type
+is used.
+
+Here are some examples of event descriptions of the previous events:
+
+```c
+{ "OHp", "pauses the execution" },
+{ "OAs(i32 cpu)", "switches it's own affinity to the CPU %{cpu}" },
+{ "OHC(i32 cpu, u64 tag)", "creates a new thread on CPU %{cpu} with tag %#llx{tag}" },
+{ "VYc+(u32 typeid, str label)", "creates task type %{typeid} with label \"%{label}\"" },
+```
+
+Which would be printed with ovnidump like:
+
+```nohighlight
+OHp  pauses the execution
+OAs  switches it's own affinity to the CPU 7
+OHC  creates a new thread on CPU 3 with tag 0x7f9239c6b6c0
+VYc  creates task type 4 with label "block computation"
+```
+
+## Model version
+
+When adding new events of changing the format of already existing
+events, the version of the model that defines the event must be changed
+accordingly to the rules of [Semantic Versions](https://semver.org).
+
+In general, adding new events will cause a minor increase in the
+version, while changing events will cause a major increase. Notice that
+the emulator will reject a stream which contains events from a model
+which is not compatible with the current model version.

doc/dev/model.md

@@ -22,7 +22,7 @@ If the model is not enabled, no other function will be called.
 
 The create function is called for each enabled model to allow them to allocate
 all the required structures to perform the emulation using the
-[extend](../extend) mechanism. All the required channels must be created and
+[extend](extend.md) mechanism. All the required channels must be created and
 registered in the patch bay in this function, so other models can found them in
 the next stage.
 

doc/dev/mux.md

@@ -1,6 +1,6 @@
 # Mux
 
-The emulator provides a mechanism to interconnect [channels](../channels) in a
+The emulator provides a mechanism to interconnect [channels](channels.md) in a
 similar way as an [analog
 multiplexer](https://en.wikipedia.org/wiki/Multiplexer) by using the `mux`
 module.
@@ -19,7 +19,7 @@ selected. This allows a multiplexer to act as a filter too.
 
 The typical use of multiplexers is to implement the tracking modes of channels.
 As an example, the following diagram shows two multiplexers used to implement
-the subsystem view of [Nanos6](../nanos6):
+the subsystem view of [Nanos6](../user/emulation/nanos6.md):
 
 ![Mux example](fig/mux.svg)
 
@@ -51,5 +51,5 @@ Multiplexers allow models to interact with each other in a controlled way. In
 the example, the blue channel (*nanos6.thread0.subsystem*) is directly modified by
 the Nanos6 model when a new event is received. While the red channels are
 controlled by the ovni model.  The rest of the channels are automatically updated
-in the propagation phase of the [bay](../patchbay) allowing the ovni model to
+in the propagation phase of the [bay](patchbay.md) allowing the ovni model to
 modify the Nanos6 Paraver view of the subsystems.

doc/dev/paraver.md

@@ -16,7 +16,7 @@ A channel can be connected to each row in a trace with `prv_register()`, so the
 new values of the channel get written in the trace. Only null and int64 data
 values are supported for now.
 
-The emission phase is controlled by the [patch bay](../patchbay) and runs all
+The emission phase is controlled by the [patch bay](patchbay.md) and runs all
 the emit callbacks at once for all dirty channels.
 
 ## Duplicate values

doc/dev/patchbay.md

@@ -1,6 +1,6 @@
 # Patch bay
 
-The patch bay (or simply bay) allows [channels](../channels/) to be registered
+The patch bay (or simply bay) allows [channels](channels.md) to be registered
 with their name so they are visible to all parts of the emulator and provides a
 way to run callbacks when the channels update their values.
 

doc/index.md

@@ -13,8 +13,9 @@ The ovni project implements a fast instrumentation library that records
 small events (starting at 12 bytes) during the execution of programs to
 later investigate how the execution happened.
 
-The instrumentation process is split in two stages: [runtime](runtime)
-tracing and [emulation](emulation/).
+The instrumentation process is split in two stages:
+[runtime](user/runtime/index.md)
+tracing and [emulation](user/emulation/index.md).
 
 During runtime, very short binary events are stored on disk which
 describe what is happening. Once the execution finishes, the events are

doc/requirements.txt

@@ -1,4 +1,4 @@
-mkdocs==1.4.1
+mkdocs==1.6.0
 markdown==3.3.7
 python-markdown-math==0.8
 jinja2==3.1.2

doc/user/concepts.md

@@ -1,31 +0,0 @@
-# Overview
-
-The objective of the ovni project is to provide insight into what
-happened at execution of a program.
-
-![Instrumentation process](fig/instrumentation.svg)
-
-The key pieces of software involved are instrumented so they emit events
-during the execution which allow the reconstruction of the execution
-later on.
-
-During the execution phase, the information gathered in the events is
-kept very short and simple, so the overhead is kept at minimum to avoid
-disturbing the execution process. Here is an example of a single event
-emitted during the execution phase, informing the current thread to
-finish the execution:
-
-	00 4f 48 65 52 c0 27 b4 d3 ec 01 00
-
-During the emulation phase, the events are read and processed in the
-emulator, reconstructing the execution. State transitions are recorded
-in a Paraver trace. Here is an example of the same thread ceasing the
-execution:
-
-	2:0:1:1:1:50105669:1:0
-
-Finally, loading the trace in the Paraver program, we can generate a
-timeline visualization of the state change. Here is the example for the
-same state transition of the thread stopping the execution:
-
-![Visualization](fig/visualization.png)

doc/user/concepts/part-model.md

@@ -0,0 +1,62 @@
+# Part model
+
+Ovni has a model to represent the hardware components as well as the software
+concepts like threads or processes. Each concept is considered to be a *part*.
+Here is an example diagram depicting the part hierarchy:
+
+![Part model](part-model.svg)
+
+Notice how a loom can restrict the CPUs of the node to its child processes.
+
+## Software parts
+
+These are not physical parts, but they abstract common concepts.
+
+### Thread
+
+A thread in ovni is directly mapped to a [POSIX
+thread](https://en.wikipedia.org/wiki/Pthreads) and they are identified by a
+`TID` which must be unique in a [node](#node). Threads in ovni have [a model with
+an internal state](../emulation/ovni.md/#thread_model) that tries to tracks the
+state of the real thread.
+
+### Process
+
+A process is directly mapped to a UNIX
+[process](https://en.wikipedia.org/wiki/Process_(computing)) and they are
+identified by a `PID` number which must be unique in a [node](#node).
+
+### Loom
+
+A loom has no direct mapping to a usual concept. It consists of a set of
+[CPUs](#cpu) from the same node and a set of processes that can *only run in
+those CPUs*. Each CPUs must belong to one and only one loom. It is often used
+to group CPUs that belong to the same process when running workloads with
+multiple processes (like with MPI).
+
+Each loom has a virtual CPU which collects running threads that are not
+exclusively assigned to a physical CPU, so we cannot determine on which CPU they
+are running.
+
+## Hardware parts
+
+These parts have a physical object assigned.
+
+### CPU
+
+A CPU is a hardware thread that can execute at most one thread at a time. Each
+CPU must have a physical ID that is unique in a node. In ovni there is also a
+virtual CPU, which simply is used to collect threads that are not tied to an
+specific physical CPU, so it cannot be easily determined where they are running.
+
+### Node
+
+A *node* refers to a compute node, often a physical machine with memory and
+network which may contain one or more
+[sockets](https://en.wikipedia.org/wiki/CPU_socket), where each socket has one
+or more CPUs.
+
+### System
+
+A system represents the complete set of hardware parts and software parts that
+are known to ovni in a given trace.

doc/user/concepts/part-model.svg

@@ -0,0 +1,516 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   width="306.91663mm"
+   height="127.00001mm"
+   viewBox="0 0 306.91663 127.00001"
+   version="1.1"
+   id="svg1"
+   inkscape:version="1.3.2 (091e20ef0f, 2023-11-25, custom)"
+   sodipodi:docname="part-model.svg"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg">
+  <sodipodi:namedview
+     id="namedview1"
+     pagecolor="#ffffff"
+     bordercolor="#999999"
+     borderopacity="1"
+     inkscape:showpageshadow="0"
+     inkscape:pageopacity="0"
+     inkscape:pagecheckerboard="0"
+     inkscape:deskcolor="#d1d1d1"
+     inkscape:document-units="mm"
+     showgrid="false"
+     inkscape:zoom="0.73049944"
+     inkscape:cx="629.70616"
+     inkscape:cy="280.62992"
+     inkscape:window-width="1920"
+     inkscape:window-height="1031"
+     inkscape:window-x="0"
+     inkscape:window-y="24"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="layer1">
+    <inkscape:grid
+       id="grid1"
+       units="px"
+       originx="-74.083327"
+       originy="-89.958345"
+       spacingx="0.26458333"
+       spacingy="0.26458334"
+       empcolor="#7a7aff"
+       empopacity="0.25098039"
+       color="#6a6aff"
+       opacity="0.1254902"
+       empspacing="5"
+       dotted="false"
+       gridanglex="30"
+       gridanglez="30"
+       visible="false" />
+  </sodipodi:namedview>
+  <defs
+     id="defs1" />
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(-74.083328,-89.958344)">
+    <rect
+       style="fill:none;stroke:none;stroke-width:0.264999;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0"
+       id="rect33"
+       width="306.91663"
+       height="127.00001"
+       x="74.083328"
+       y="89.958344" />
+    <rect
+       style="fill:#ffeeaa;stroke:#000000;stroke-width:0.2;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0"
+       id="rect32"
+       width="121.70832"
+       height="84.666679"
+       x="253.99998"
+       y="127.00001" />
+    <text
+       xml:space="preserve"
+       style="font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
+       x="290.19363"
+       y="202.25124"
+       id="text32"><tspan
+         sodipodi:role="line"
+         id="tspan32"
+         style="font-size:6.35px;stroke-width:0.264583"
+         x="290.19363"
+         y="202.25124">Hardware parts</tspan></text>
+    <rect
+       style="fill:#eeffaa;stroke:#000000;stroke-width:0.2;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0"
+       id="rect31"
+       width="169.33333"
+       height="84.666679"
+       x="79.374992"
+       y="127.00001" />
+    <rect
+       style="fill:#fff6d5;stroke:#000000;stroke-width:0.3;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0"
+       id="rect27"
+       width="47.625004"
+       height="21.166676"
+       x="269.87497"
+       y="148.16667"
+       ry="2.645834" />
+    <rect
+       style="fill:#ffffff;stroke:#000000;stroke-width:0.2;stroke-linecap:square;stroke-dasharray:none"
+       id="rect1"
+       width="21.166662"
+       height="10.583329"
+       x="84.566666"
+       y="174.52502"
+       ry="0" />
+    <text
+       xml:space="preserve"
+       style="font-size:4.23333px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
+       x="87.947342"
+       y="181.39488"
+       id="text1"><tspan
+         sodipodi:role="line"
+         id="tspan1"
+         style="stroke-width:0.264583"
+         x="87.947342"
+         y="181.39488">Thread</tspan></text>
+    <rect
+       style="fill:#ffffff;stroke:#000000;stroke-width:0.2;stroke-linecap:square;stroke-dasharray:none"
+       id="rect3"
+       width="21.166664"
+       height="10.583337"
+       x="111.12499"
+       y="153.45834"
+       ry="0" />
+    <text
+       xml:space="preserve"
+       style="font-size:4.23333px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
+       x="113.57446"
+       y="160.26309"
+       id="text3"><tspan
+         sodipodi:role="line"
+         id="tspan3"
+         style="stroke-width:0.264583"
+         x="113.57446"
+         y="160.26309">Process</tspan></text>
+    <rect
+       style="fill:#ffffff;stroke:#000000;stroke-width:0.2;stroke-linecap:square;stroke-dasharray:none"
+       id="rect4"
+       width="21.166664"
+       height="10.583333"
+       x="111.12499"
+       y="174.62502"
+       ry="0" />
+    <text
+       xml:space="preserve"
+       style="font-size:4.23333px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
+       x="114.50567"
+       y="181.49487"
+       id="text4"><tspan
+         sodipodi:role="line"
+         id="tspan4"
+         style="stroke-width:0.264583"
+         x="114.50567"
+         y="181.49487">Thread</tspan></text>
+    <rect
+       style="fill:#ffffff;stroke:#000000;stroke-width:0.2;stroke-linecap:square;stroke-dasharray:none"
+       id="rect5"
+       width="21.166658"
+       height="10.583333"
+       x="137.58333"
+       y="174.62502"
+       ry="0" />
+    <text
+       xml:space="preserve"
+       style="font-size:4.23333px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
+       x="140.964"
+       y="181.49487"
+       id="text5"><tspan
+         sodipodi:role="line"
+         id="tspan5"
+         style="stroke-width:0.264583"
+         x="140.964"
+         y="181.49487">Thread</tspan></text>
+    <rect
+       style="fill:#ffffff;stroke:#000000;stroke-width:0.2;stroke-linecap:square;stroke-dasharray:none"
+       id="rect6"
+       width="21.166672"
+       height="10.583333"
+       x="169.33331"
+       y="174.62502"
+       ry="0" />
+    <text
+       xml:space="preserve"
+       style="font-size:4.23333px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
+       x="172.71399"
+       y="181.49487"
+       id="text6"><tspan
+         sodipodi:role="line"
+         id="tspan6"
+         style="stroke-width:0.264583"
+         x="172.71399"
+         y="181.49487">Thread</tspan></text>
+    <rect
+       style="fill:#ffffff;stroke:#000000;stroke-width:0.2;stroke-linecap:square;stroke-dasharray:none"
+       id="rect7"
+       width="20.966642"
+       height="10.38336"
+       x="195.99167"
+       y="153.55833"
+       ry="0" />
+    <text
+       xml:space="preserve"
+       style="font-size:4.23333px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
+       x="198.24112"
+       y="160.16312"
+       id="text7"><tspan
+         sodipodi:role="line"
+         id="tspan7"
+         style="stroke-width:0.264583"
+         x="198.24112"
+         y="160.16312">Process</tspan></text>
+    <rect
+       style="fill:#ffffff;stroke:#000000;stroke-width:0.2;stroke-linecap:square;stroke-dasharray:none"
+       id="rect8"
+       width="21.16667"
+       height="10.583333"
+       x="195.79164"
+       y="174.62502"
+       ry="0" />
+    <text
+       xml:space="preserve"
+       style="font-size:4.23333px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
+       x="199.17232"
+       y="181.49487"
+       id="text8"><tspan
+         sodipodi:role="line"
+         id="tspan8"
+         style="stroke-width:0.264583"
+         x="199.17232"
+         y="181.49487">Thread</tspan></text>
+    <rect
+       style="fill:#ffffff;stroke:#000000;stroke-width:0.2;stroke-linecap:square;stroke-dasharray:none"
+       id="rect9"
+       width="21.166666"
+       height="10.583333"
+       x="222.24997"
+       y="174.62502"
+       ry="0" />
+    <text
+       xml:space="preserve"
+       style="font-size:4.23333px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
+       x="225.63065"
+       y="181.49487"
+       id="text9"><tspan
+         sodipodi:role="line"
+         id="tspan9"
+         style="stroke-width:0.264583"
+         x="225.63065"
+         y="181.49487">Thread</tspan></text>
+    <rect
+       style="fill:#ffffff;stroke:#000000;stroke-width:0.2;stroke-linecap:square;stroke-dasharray:none"
+       id="rect10"
+       width="21.166658"
+       height="10.583337"
+       x="153.45833"
+       y="132.29167"
+       ry="0" />
+    <text
+       xml:space="preserve"
+       style="font-size:4.23333px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
+       x="158.21979"
+       y="139.09644"
+       id="text10"><tspan
+         sodipodi:role="line"
+         id="tspan10"
+         style="stroke-width:0.264583"
+         x="158.21979"
+         y="139.09644">Loom</tspan></text>
+    <rect
+       style="fill:#ffffff;stroke:#000000;stroke-width:0.2;stroke-linecap:square;stroke-dasharray:none"
+       id="rect11"
+       width="21.166651"
+       height="10.58334"
+       x="306.91666"
+       y="132.29167"
+       ry="0" />
+    <text
+       xml:space="preserve"
+       style="font-size:4.23333px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
+       x="311.88068"
+       y="139.16154"
+       id="text11"><tspan
+         sodipodi:role="line"
+         id="tspan11"
+         style="stroke-width:0.264583"
+         x="311.88068"
+         y="139.16154">Node</tspan></text>
+    <rect
+       style="fill:#ffffff;stroke:#000000;stroke-width:0.2;stroke-linecap:square;stroke-dasharray:none"
+       id="rect12"
+       width="15.875001"
+       height="10.583341"
+       x="275.16666"
+       y="153.45834"
+       ry="0" />
+    <text
+       xml:space="preserve"
+       style="font-size:4.23333px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
+       x="278.86566"
+       y="160.291"
+       id="text12"><tspan
+         sodipodi:role="line"
+         id="tspan12"
+         style="stroke-width:0.264583"
+         x="278.86566"
+         y="160.291">CPU</tspan></text>
+    <rect
+       style="fill:#ffffff;stroke:#000000;stroke-width:0.2;stroke-linecap:square;stroke-dasharray:none"
+       id="rect13"
+       width="15.874983"
+       height="10.583341"
+       x="296.33334"
+       y="153.45834"
+       ry="0" />
+    <text
+       xml:space="preserve"
+       style="font-size:4.23333px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
+       x="300.03235"
+       y="160.291"
+       id="text13"><tspan
+         sodipodi:role="line"
+         id="tspan13"
+         style="stroke-width:0.264583"
+         x="300.03235"
+         y="160.291">CPU</tspan></text>
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       d="m 121.70832,164.04168 0,10.58334"
+       id="path15"
+       inkscape:connector-type="polyline"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect3"
+       inkscape:connection-end="#rect4" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       d="m 206.45046,163.94169 -0.0505,10.68333"
+       id="path16"
+       inkscape:connector-type="polyline"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect7"
+       inkscape:connection-end="#rect8" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       d="m 212.94006,163.94169 13.30366,10.68333"
+       id="path17"
+       inkscape:connector-type="polyline"
+       inkscape:connector-curvature="0"
+       inkscape:connection-end="#rect9"
+       inkscape:connection-start="#rect7" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       d="m 199.96086,163.94169 -13.40463,10.68333"
+       id="path18"
+       inkscape:connector-type="polyline"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect7"
+       inkscape:connection-end="#rect6" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       d="m 128.32291,164.04168 13.22917,10.58334"
+       id="path19"
+       inkscape:connector-type="polyline"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect3"
+       inkscape:connection-end="#rect5" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       d="m 115.03723,164.04168 -13.21614,10.48334"
+       id="path20"
+       inkscape:connector-type="polyline"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect3"
+       inkscape:connection-end="#rect1" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       d="m 153.45833,142.87501 -21.16667,10.58333"
+       id="path21"
+       inkscape:connector-type="polyline"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect10"
+       inkscape:connection-end="#rect3" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       d="m 174.62499,142.86254 21.44212,10.69579"
+       id="path22"
+       inkscape:connector-type="polyline"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect10"
+       inkscape:connection-end="#rect7" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       d="m 314.19269,142.87501 -6.61457,10.58333"
+       id="path24"
+       inkscape:connector-type="polyline"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect11"
+       inkscape:connection-end="#rect13" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       d="M 308.90102,142.87501 291.04166,153.8654"
+       id="path25"
+       inkscape:connector-type="polyline"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect11"
+       inkscape:connection-end="#rect12" />
+    <rect
+       style="fill:#ffffff;stroke:#000000;stroke-width:0.2;stroke-linecap:square;stroke-dasharray:none"
+       id="rect25"
+       width="15.875013"
+       height="10.583337"
+       x="322.79163"
+       y="153.45834"
+       ry="0" />
+    <text
+       xml:space="preserve"
+       style="font-size:4.23333px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
+       x="326.49063"
+       y="160.291"
+       id="text25"><tspan
+         sodipodi:role="line"
+         id="tspan25"
+         style="stroke-width:0.264583"
+         x="326.49063"
+         y="160.291">CPU</tspan></text>
+    <rect
+       style="fill:#ffffff;stroke:#000000;stroke-width:0.2;stroke-linecap:square;stroke-dasharray:none"
+       id="rect26"
+       width="15.874991"
+       height="10.583337"
+       x="343.95831"
+       y="153.45834"
+       ry="0" />
+    <text
+       xml:space="preserve"
+       style="font-size:4.23333px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
+       x="347.65732"
+       y="160.291"
+       id="text26"><tspan
+         sodipodi:role="line"
+         id="tspan26"
+         style="stroke-width:0.264583"
+         x="347.65732"
+         y="160.291">CPU</tspan></text>
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       d="m 320.80727,142.87501 6.61457,10.58333"
+       id="path26"
+       inkscape:connector-type="polyline"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect11"
+       inkscape:connection-end="#rect25" />
+    <path
+       style="display:inline;fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       d="m 326.09894,142.87501 17.85937,10.99039"
+       id="path27"
+       inkscape:connector-type="polyline"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect11"
+       inkscape:connection-end="#rect26" />
+    <path
+       style="display:inline;fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.265;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
+       d="m 174.62499,139.70001 95.24998,19.05"
+       id="path28"
+       inkscape:connector-type="polyline"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect10" />
+    <rect
+       style="fill:#ffffff;stroke:#000000;stroke-width:0.2;stroke-linecap:square;stroke-dasharray:none"
+       id="rect29"
+       width="21.166666"
+       height="10.583335"
+       x="216.95831"
+       y="95.250008"
+       ry="0" />
+    <text
+       xml:space="preserve"
+       style="font-size:4.23333px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
+       x="219.6889"
+       y="101.67236"
+       id="text29"><tspan
+         sodipodi:role="line"
+         id="tspan29"
+         style="stroke-width:0.264583"
+         x="219.6889"
+         y="101.67236">System</tspan></text>
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       d="m 238.12498,104.89952 68.79168,28.32598"
+       id="path29"
+       inkscape:connector-type="polyline"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect29"
+       inkscape:connection-end="#rect11" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       d="m 218.47022,105.83334 -45.35713,26.45833"
+       id="path30"
+       inkscape:connector-type="polyline"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect29"
+       inkscape:connection-end="#rect10" />
+    <text
+       xml:space="preserve"
+       style="font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
+       x="140.81671"
+       y="202.25124"
+       id="text31"><tspan
+         sodipodi:role="line"
+         id="tspan31"
+         style="font-size:6.35px;stroke-width:0.264583"
+         x="140.81671"
+         y="202.25124">Software parts</tspan></text>
+  </g>
+</svg>

doc/user/concepts/trace.md

@@ -0,0 +1,123 @@
+# Trace concepts
+
+When using libovni to generate traces or the emulator to process them, there are
+several concepts to keep in mind.
+
+## Trace elements
+
+The information generated by a program or later processed by other ovni tools is
+known as a trace. A runtime trace stores the information as-is in disk from a
+program execution. While a emulation trace is generated from the runtime trace
+for visualization with Paraver.
+
+Both runtime and emulation traces are always stored inside the same directory,
+by default `ovni/`, which is known as the *trace directory*.
+
+Here are the components of a runtime trace, as generated by libovni:
+
+<p align="center">
+<img alt="Trace concepts" src="../trace.svg">
+</p>
+
+### Stream
+
+A stream is a directory which contains a binary stream and the associated stream
+metadata file. Each stream is associated with a given part of a system. As of
+now, libovni can only generate streams associated to [threads](part-model.md#thread).
+
+### Stream metadata
+
+The stream metadata is a JSON file named `stream.json` which holds information
+about the stream itself.
+
+### Binary stream
+
+A binary stream is a file named `stream.obs` (.obs stands for Ovni Binary
+Stream) composed of a header and a concatenated array of events without padding.
+Notice that each event may have different length.
+
+### Event
+
+An event is a point in time that has some information associated. Events written
+at runtime by libovni have at MCV, a clock and a optional payload. The list of
+all events recognized by the emulator can be found [here](../emulation/events.md).
+
+Events can be displayed by ovnidump, which shows an explanation of what the
+event means:
+
+```txt
+$ ovnidump ovni/loom.hop.nosv-u1000/proc.1121064 | grep -A 10 VTx | head
+517267929632815  VTx  thread.1121064  executes the task 1 with bodyid 0
+517267930261672  VYc  thread.1121064  creates task type 2 with label "task"
+517267930875858  VTC  thread.1121064  creates parallel task 2 with type 2
+517267930877789  VU[  thread.1121064  starts submitting a task
+517267930877990  VU]  thread.1121064  stops  submitting a task
+517267930878098  VTC  thread.1121064  creates parallel task 3 with type 2
+517267930878196  VU[  thread.1121064  starts submitting a task
+517267930878349  VU]  thread.1121064  stops  submitting a task
+517267930878432  VTC  thread.1121064  creates parallel task 4 with type 2
+517267930878494  VU[  thread.1121064  starts submitting a task
+```
+
+There are two types or events: normal and jumbo events, the latter can hold
+large attached payloads.
+
+### MCV
+
+The MCV acronym is the abbreviation of Model-Class-Value, which are a three
+characters that identify any event. The MCV is shown in the ovnitop and ovnidump
+tools and allows easy filtering with grep, for a single or related events:
+
+```
+$ ovnitop ovni | grep VT
+VTe      20002
+VTx      20002
+VTC        200
+VTc          2
+VTp          1
+VTr          1
+```
+
+### Clock
+
+A clock is a 64 bit counter, which counts the number of nanoseconds from an
+arbitrary point in time in the past. Each event has the value of the clock
+stored inside, to indicate when that event happened. In a given trace there can
+be multiple clocks which don't refer to the same point in the past and must be
+corrected so they all produce an ordered sequence of events. The ovnisync
+program performs this correction by measuring the difference across clocks of
+different nodes.
+
+### Payload
+
+Events may have associated additional information which is stored in the stream.
+Normal events can hold up to 16 bytes, otherwise the jumbo events must be used
+to hold additional payload.
+
+## Other related concepts
+
+Apart from the trace itself, there are other concepts to keep in mind when the
+trace is being processed by the emulator.
+
+### Event model
+
+Each event belongs to an event model, as identified by the model character in
+the MCV. An event model is composed of several components:
+
+- A set of [events](#event) all with the same model identifier in the
+  [MCV](#mcv)
+- The emulator code that processes those events.
+- A human readable name, like `ovni` or `nanos6`.
+- A semantic version.
+
+### State
+
+A state is a discrete value that can change over time based on the events the
+emulator receives. Usually a single event causes a single state change, which is
+then written to the Paraver traces. An example is the thread state, which can
+change over time based on the events `OH*` that indicate a state transition
+of the current thread.
+
+In contrast with an event, states have a duration associated which can usually
+be observed in Paraver. Notice that the trace only contains events, the states
+are computed at emulation.

doc/user/concepts/trace.svg

@@ -0,0 +1,474 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   width="116.41666mm"
+   height="105.83334mm"
+   viewBox="0 0 116.41666 105.83334"
+   version="1.1"
+   id="svg1"
+   inkscape:version="1.3.2 (091e20ef0f, 2023-11-25, custom)"
+   sodipodi:docname="trace.svg"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg">
+  <sodipodi:namedview
+     id="namedview1"
+     pagecolor="#ffffff"
+     bordercolor="#999999"
+     borderopacity="1"
+     inkscape:showpageshadow="0"
+     inkscape:pageopacity="0"
+     inkscape:pagecheckerboard="0"
+     inkscape:deskcolor="#d1d1d1"
+     inkscape:document-units="mm"
+     showgrid="true"
+     inkscape:zoom="1.4609989"
+     inkscape:cx="248.80238"
+     inkscape:cy="199.17879"
+     inkscape:window-width="1920"
+     inkscape:window-height="1031"
+     inkscape:window-x="0"
+     inkscape:window-y="24"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="layer1">
+    <inkscape:grid
+       id="grid1"
+       units="px"
+       originx="-39.687498"
+       originy="-15.875"
+       spacingx="0.26458333"
+       spacingy="0.26458334"
+       empcolor="#7a7aff"
+       empopacity="0.25098039"
+       color="#6a6aff"
+       opacity="0.1254902"
+       empspacing="5"
+       dotted="false"
+       gridanglex="30"
+       gridanglez="30"
+       visible="true" />
+  </sodipodi:namedview>
+  <defs
+     id="defs1">
+    <rect
+       x="45"
+       y="370"
+       width="209.99998"
+       height="35"
+       id="rect15" />
+  </defs>
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(-39.687498,-15.875)">
+    <g
+       id="g59"
+       transform="translate(5.5781353,-0.74017783)"
+       style="stroke-width:0.15875;stroke-dasharray:none">
+      <text
+         xml:space="preserve"
+         style="font-size:4.23333px;line-height:1.25;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;stroke-width:0.15875;stroke-dasharray:none"
+         x="100.23113"
+         y="24.896034"
+         id="text22"><tspan
+           sodipodi:role="line"
+           id="tspan22"
+           style="text-align:center;text-anchor:middle;stroke-width:0.15875;stroke-dasharray:none"
+           x="100.23113"
+           y="24.896034">Trace</tspan></text>
+      <rect
+         style="fill:none;stroke:#000000;stroke-width:0.15875;stroke-linecap:square;stroke-dasharray:none"
+         id="rect32"
+         width="15.875"
+         height="7.9375005"
+         x="92.281235"
+         y="19.4142" />
+    </g>
+    <g
+       id="g57"
+       transform="translate(-0.84260657,-0.98969722)"
+       style="stroke-width:0.15875;stroke-dasharray:none">
+      <text
+         xml:space="preserve"
+         style="font-size:4.23333px;line-height:1.25;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;stroke-width:0.15875;stroke-dasharray:none"
+         x="106.39689"
+         y="46.186943"
+         id="text24"><tspan
+           sodipodi:role="line"
+           id="tspan24"
+           style="text-align:center;text-anchor:middle;stroke-width:0.15875;stroke-dasharray:none"
+           x="106.39689"
+           y="46.186943">Stream</tspan></text>
+      <rect
+         style="fill:none;stroke:#000000;stroke-width:0.15875;stroke-linecap:square;stroke-dasharray:none"
+         id="rect33"
+         width="21.166666"
+         height="7.9375019"
+         x="96.092606"
+         y="40.6772" />
+    </g>
+    <g
+       id="g56"
+       transform="translate(0.32292488,-0.89336269)"
+       style="stroke-width:0.15875;stroke-dasharray:none">
+      <text
+         xml:space="preserve"
+         style="font-size:4.23333px;line-height:1.25;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;stroke-width:0.15875;stroke-dasharray:none"
+         x="78.773026"
+         y="46.090603"
+         id="text23"><tspan
+           sodipodi:role="line"
+           id="tspan23"
+           style="text-align:center;text-anchor:middle;stroke-width:0.15875;stroke-dasharray:none"
+           x="78.773026"
+           y="46.090603">Stream</tspan></text>
+      <rect
+         style="fill:none;stroke:#000000;stroke-width:0.15875;stroke-linecap:square;stroke-dasharray:none"
+         id="rect34"
+         width="21.166666"
+         height="7.9375"
+         x="68.468742"
+         y="40.580864" />
+    </g>
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.15875;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1"
+       d="M 100.80665,26.611522 84.365216,39.687501"
+       id="path35"
+       inkscape:connector-type="polyline"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect32"
+       inkscape:connection-end="#rect34" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.15875;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1"
+       d="m 105.80376,26.611522 0.0227,13.075981"
+       id="path36"
+       inkscape:connector-type="polyline"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect32"
+       inkscape:connection-end="#rect33" />
+    <g
+       id="g54"
+       style="stroke-width:0.15875;stroke-dasharray:none"
+       transform="translate(-0.99999828,-3.5391978)">
+      <text
+         xml:space="preserve"
+         style="font-size:4.23333px;line-height:1.25;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;stroke-width:0.15875;stroke-dasharray:none"
+         x="58.792854"
+         y="72.586143"
+         id="text25"><tspan
+           sodipodi:role="line"
+           id="tspan25"
+           style="text-align:center;text-anchor:middle;stroke-width:0.15875;stroke-dasharray:none"
+           x="58.792854"
+           y="72.586143">Metadata</tspan></text>
+      <rect
+         style="fill:none;stroke:#000000;stroke-width:0.15875;stroke-linecap:square;stroke-dasharray:none"
+         id="rect36"
+         width="26.458336"
+         height="7.9375"
+         x="45.979164"
+         y="67.0392" />
+    </g>
+    <g
+       id="g55"
+       style="stroke-width:0.15875;stroke-dasharray:none"
+       transform="translate(-0.99999828,-3.5391978)">
+      <text
+         xml:space="preserve"
+         style="font-size:4.23333px;line-height:1.25;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;stroke-width:0.15875;stroke-dasharray:none"
+         x="97.157433"
+         y="72.175835"
+         id="text26"><tspan
+           sodipodi:role="line"
+           id="tspan26"
+           style="text-align:center;text-anchor:middle;stroke-width:0.15875;stroke-dasharray:none"
+           x="97.157433"
+           y="72.175835">Binary stream</tspan></text>
+      <rect
+         style="fill:none;stroke:#000000;stroke-width:0.15875;stroke-linecap:square;stroke-dasharray:none"
+         id="rect37"
+         width="34.395832"
+         height="7.9375"
+         x="80.374992"
+         y="67.0392" />
+    </g>
+    <rect
+       style="stroke-width:0.15875;stroke-dasharray:none;fill:none;stroke:none;stroke-linecap:square"
+       id="rect60"
+       width="116.41666"
+       height="105.83334"
+       x="40.687496"
+       y="19.414198"
+       inkscape:label="background"
+       transform="translate(-0.99999735,-3.5391979)" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.15875;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1"
+       d="M 75.847223,47.625001 61.736111,63.500002"
+       id="path37"
+       inkscape:connector-type="polyline"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect34"
+       inkscape:connection-end="#rect36" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.15875;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1"
+       d="M 82.241318,47.625001 93.706592,63.500002"
+       id="path38"
+       inkscape:connector-type="polyline"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect34"
+       inkscape:connection-end="#rect37" />
+    <g
+       id="g53"
+       style="stroke-width:0.15875;stroke-dasharray:none"
+       transform="translate(-8.2507438,-3.0742007)">
+      <text
+         xml:space="preserve"
+         style="font-size:4.23333px;line-height:1.25;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;stroke-width:0.15875;stroke-dasharray:none"
+         x="62.074863"
+         y="95.933655"
+         id="text27"><tspan
+           sodipodi:role="line"
+           id="tspan27"
+           style="text-align:center;text-anchor:middle;stroke-width:0.15875;stroke-dasharray:none"
+           x="62.074863"
+           y="95.933655">Header</tspan></text>
+      <rect
+         style="fill:none;stroke:#000000;stroke-width:0.15875;stroke-linecap:square;stroke-dasharray:none"
+         id="rect38"
+         width="23.812506"
+         height="7.9375014"
+         x="50.584076"
+         y="90.386703" />
+    </g>
+    <g
+       id="g52"
+       transform="translate(-8.2507438,-3.3066989)"
+       style="stroke-width:0.15875;stroke-dasharray:none">
+      <text
+         xml:space="preserve"
+         style="font-size:4.23333px;line-height:1.25;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;stroke-width:0.15875;stroke-dasharray:none"
+         x="91.411552"
+         y="96.101036"
+         id="text28"><tspan
+           sodipodi:role="line"
+           id="tspan28"
+           style="text-align:center;text-anchor:middle;stroke-width:0.15875;stroke-dasharray:none"
+           x="91.411552"
+           y="96.101036">Event</tspan></text>
+      <rect
+         style="fill:none;stroke:#000000;stroke-width:0.15875;stroke-linecap:square;stroke-dasharray:none"
+         id="rect39"
+         width="23.812506"
+         height="7.9375014"
+         x="79.920769"
+         y="90.619202" />
+    </g>
+    <g
+       id="g51"
+       style="stroke-width:0.15875;stroke-dasharray:none"
+       transform="translate(-8.2507438,-3.0742007)">
+      <text
+         xml:space="preserve"
+         style="font-size:4.23333px;line-height:1.25;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;stroke-width:0.15875;stroke-dasharray:none"
+         x="120.28321"
+         y="95.868538"
+         id="text29"><tspan
+           sodipodi:role="line"
+           id="tspan29"
+           style="text-align:center;text-anchor:middle;stroke-width:0.15875;stroke-dasharray:none"
+           x="120.28321"
+           y="95.868538">Event</tspan></text>
+      <rect
+         style="fill:none;stroke:#000000;stroke-width:0.15875;stroke-linecap:square;stroke-dasharray:none"
+         id="rect40"
+         width="23.812506"
+         height="7.9375014"
+         x="108.79243"
+         y="90.386703" />
+    </g>
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.15875;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1"
+       d="M 89.517356,71.437502 61.29514,87.312503"
+       id="path40"
+       inkscape:connector-type="polyline"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect37"
+       inkscape:connection-end="#rect38" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.15875;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1"
+       d="M 94.406805,71.437502 85.742383,87.312503"
+       id="path41"
+       inkscape:connector-type="polyline"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect37"
+       inkscape:connection-end="#rect39" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.15875;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1"
+       d="M 99.218748,71.437502 109.8021,87.312503"
+       id="path42"
+       inkscape:connector-type="polyline"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect37"
+       inkscape:connection-end="#rect40" />
+    <g
+       id="g46"
+       transform="translate(-68.659367,2.7781237)"
+       style="stroke-width:0.15875;stroke-dasharray:none">
+      <text
+         xml:space="preserve"
+         style="font-size:4.23333px;line-height:1.25;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;stroke-width:0.15875;stroke-dasharray:none"
+         x="123.93869"
+         y="113.98891"
+         id="text30"><tspan
+           sodipodi:role="line"
+           id="tspan30"
+           style="text-align:center;text-anchor:middle;stroke-width:0.15875;stroke-dasharray:none"
+           x="123.93869"
+           y="113.98891">MCV</tspan></text>
+      <rect
+         style="fill:none;stroke:#000000;stroke-width:0.15875;stroke-linecap:square;stroke-dasharray:none"
+         id="rect42"
+         width="15.875001"
+         height="7.9375038"
+         x="116.41666"
+         y="108.47917" />
+    </g>
+    <g
+       id="g45"
+       transform="translate(-68.659367,5.4239515)"
+       style="stroke-width:0.15875;stroke-dasharray:none">
+      <text
+         xml:space="preserve"
+         style="font-size:4.23333px;line-height:1.25;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;stroke-width:0.15875;stroke-dasharray:none"
+         x="147.92894"
+         y="111.38029"
+         id="text31"><tspan
+           sodipodi:role="line"
+           id="tspan31"
+           style="text-align:center;text-anchor:middle;stroke-width:0.15875;stroke-dasharray:none"
+           x="147.92894"
+           y="111.38029">Clock</tspan></text>
+      <rect
+         style="fill:none;stroke:#000000;stroke-width:0.15875;stroke-linecap:square;stroke-dasharray:none"
+         id="rect43"
+         width="15.875001"
+         height="7.9375038"
+         x="140.22916"
+         y="105.83334" />
+    </g>
+    <g
+       id="g44"
+       transform="translate(-66.112857,13.610614)"
+       style="stroke-width:0.15875;stroke-dasharray:none">
+      <g
+         id="g60"
+         style="stroke-width:0.15875;stroke-dasharray:none">
+        <text
+           xml:space="preserve"
+           style="font-size:4.23333px;line-height:1.25;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;stroke-width:0.15875;stroke-dasharray:none"
+           x="172.85362"
+           y="102.65102"
+           id="text32"><tspan
+             sodipodi:role="line"
+             id="tspan32"
+             style="text-align:center;text-anchor:middle;stroke-width:0.15875;stroke-dasharray:none"
+             x="172.85362"
+             y="102.65102">Payload</tspan></text>
+        <rect
+           style="fill:none;stroke:#000000;stroke-width:0.15875;stroke-linecap:square;stroke-dasharray:none"
+           id="rect44"
+           width="23.812506"
+           height="7.9375014"
+           x="161.36285"
+           y="97.514389" />
+      </g>
+    </g>
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.15875;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1"
+       d="M 83.366344,95.250004 80.27399,111.2573"
+       id="path47"
+       inkscape:connector-type="polyline"
+       inkscape:connector-curvature="0"
+       inkscape:connection-end="#g45" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.15875;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1"
+       d="M 80.082847,95.250004 60.540101,111.2573"
+       id="path48"
+       inkscape:connector-type="polyline"
+       inkscape:connector-curvature="0"
+       inkscape:connection-end="#g46" />
+    <g
+       id="g58"
+       transform="translate(-1.9073076,-1.4153422)"
+       style="stroke-width:0.15875;stroke-dasharray:none">
+      <text
+         xml:space="preserve"
+         style="font-size:4.23333px;line-height:1.25;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;stroke-width:0.15875;stroke-dasharray:none"
+         x="132.47266"
+         y="46.496338"
+         id="text49"><tspan
+           sodipodi:role="line"
+           id="tspan49"
+           style="text-align:center;text-anchor:middle;stroke-width:0.15875;stroke-dasharray:none"
+           x="132.47266"
+           y="46.496338">...</tspan></text>
+      <rect
+         style="fill:none;stroke:#000000;stroke-width:0.15875;stroke-linecap:square;stroke-dasharray:none"
+         id="rect49"
+         width="18.619415"
+         height="7.9374995"
+         x="123.61564"
+         y="41.102844" />
+    </g>
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.15875;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1"
+       d="m 110.56031,26.611522 15.69429,13.07598"
+       id="path49"
+       inkscape:connector-type="polyline"
+       inkscape:connector-curvature="0"
+       inkscape:connection-end="#rect49"
+       inkscape:connection-start="#rect32" />
+    <g
+       id="g50"
+       transform="translate(-8.2507438,-2.9293636)"
+       style="stroke-width:0.15875;stroke-dasharray:none">
+      <text
+         xml:space="preserve"
+         style="font-size:4.23333px;line-height:1.25;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;stroke-width:0.15875;stroke-dasharray:none"
+         x="149.35014"
+         y="95.635361"
+         id="text50"><tspan
+           sodipodi:role="line"
+           id="tspan50"
+           style="text-align:center;text-anchor:middle;stroke-width:0.15875;stroke-dasharray:none"
+           x="149.35014"
+           y="95.635361">...</tspan></text>
+      <rect
+         style="fill:none;stroke:#000000;stroke-width:0.15875;stroke-linecap:square;stroke-dasharray:none"
+         id="rect40-5"
+         width="23.812506"
+         height="7.9375014"
+         x="137.89658"
+         y="90.241867" />
+    </g>
+    <path
+       style="display:inline;fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.15875;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1"
+       d="m 104.06944,71.437502 29.98612,15.875001"
+       id="path50"
+       inkscape:connector-type="polyline"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect37"
+       inkscape:connection-end="#g50" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.15875;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1"
+       d="M 87.506274,95.250004 103.22625,111.125"
+       id="path60"
+       inkscape:connector-type="polyline"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#g52"
+       inkscape:connection-end="#g44" />
+  </g>
+</svg>

doc/user/emulation/fig/body-model.dot

@@ -0,0 +1,9 @@
+digraph {
+	graph [size="6!", ranksep="0.7", nodesep="1"];
+
+	Created -> Running;
+	Running -> Paused [label=BODY_FLAG_PAUSE];
+	Paused -> Running;
+	Running -> Dead;
+	Dead -> Running [label="BODY_FLAG_RESURRECT\niteration++"];
+}

doc/user/emulation/fig/body-model.svg

@@ -0,0 +1,69 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!-- Generated by graphviz version 8.1.0 (0)
+ -->
+<!-- Pages: 1 -->
+<svg width="432pt" height="202pt"
+ viewBox="0.00 0.00 432.00 202.47" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<g id="graph0" class="graph" transform="scale(0.805048 0.805048) rotate(0) translate(4 247.5)">
+<polygon fill="white" stroke="none" points="-4,4 -4,-247.5 532.61,-247.5 532.61,4 -4,4"/>
+<!-- Created -->
+<g id="node1" class="node">
+<title>Created</title>
+<ellipse fill="none" stroke="black" cx="246.61" cy="-225.5" rx="51.21" ry="18"/>
+<text text-anchor="middle" x="246.61" y="-220.82" font-family="Times,serif" font-size="14.00">Created</text>
+</g>
+<!-- Running -->
+<g id="node2" class="node">
+<title>Running</title>
+<ellipse fill="none" stroke="black" cx="246.61" cy="-138.5" rx="53.31" ry="18"/>
+<text text-anchor="middle" x="246.61" y="-133.82" font-family="Times,serif" font-size="14.00">Running</text>
+</g>
+<!-- Created&#45;&gt;Running -->
+<g id="edge1" class="edge">
+<title>Created&#45;&gt;Running</title>
+<path fill="none" stroke="black" d="M246.61,-207.3C246.61,-195.97 246.61,-180.86 246.61,-167.79"/>
+<polygon fill="black" stroke="black" points="250.11,-167.97 246.61,-157.97 243.11,-167.97 250.11,-167.97"/>
+</g>
+<!-- Paused -->
+<g id="node3" class="node">
+<title>Paused</title>
+<ellipse fill="none" stroke="black" cx="132.61" cy="-18" rx="47" ry="18"/>
+<text text-anchor="middle" x="132.61" y="-13.32" font-family="Times,serif" font-size="14.00">Paused</text>
+</g>
+<!-- Running&#45;&gt;Paused -->
+<g id="edge2" class="edge">
+<title>Running&#45;&gt;Paused</title>
+<path fill="none" stroke="black" d="M193.44,-135.87C130.26,-132.68 30.43,-123.26 7.11,-95.5 -2.75,-83.76 -1.84,-73.44 7.11,-61 22.79,-39.22 50.29,-28.57 75.56,-23.43"/>
+<polygon fill="black" stroke="black" points="75.98,-26.73 85.22,-21.55 74.76,-19.84 75.98,-26.73"/>
+<text text-anchor="middle" x="80.36" y="-73.58" font-family="Times,serif" font-size="14.00">BODY_FLAG_PAUSE</text>
+</g>
+<!-- Dead -->
+<g id="node4" class="node">
+<title>Dead</title>
+<ellipse fill="none" stroke="black" cx="343.61" cy="-18" rx="37" ry="18"/>
+<text text-anchor="middle" x="343.61" y="-13.32" font-family="Times,serif" font-size="14.00">Dead</text>
+</g>
+<!-- Running&#45;&gt;Dead -->
+<g id="edge4" class="edge">
+<title>Running&#45;&gt;Dead</title>
+<path fill="none" stroke="black" d="M260.31,-120.77C276.79,-100.63 304.62,-66.64 323.53,-43.53"/>
+<polygon fill="black" stroke="black" points="326.68,-46.22 330.3,-36.26 321.26,-41.78 326.68,-46.22"/>
+</g>
+<!-- Paused&#45;&gt;Running -->
+<g id="edge3" class="edge">
+<title>Paused&#45;&gt;Running</title>
+<path fill="none" stroke="black" d="M148.33,-35.34C167.75,-55.52 200.96,-90.04 223.34,-113.31"/>
+<polygon fill="black" stroke="black" points="220.31,-115.24 229.76,-120.02 225.35,-110.39 220.31,-115.24"/>
+</g>
+<!-- Dead&#45;&gt;Running -->
+<g id="edge5" class="edge">
+<title>Dead&#45;&gt;Running</title>
+<path fill="none" stroke="black" d="M343.24,-36.22C342.05,-52.96 337.96,-78.31 324.61,-95.5 316.51,-105.94 305.06,-114.1 293.42,-120.34"/>
+<polygon fill="black" stroke="black" points="292.09,-117.59 284.66,-125.14 295.18,-123.87 292.09,-117.59"/>
+<text text-anchor="middle" x="434.11" y="-82.2" font-family="Times,serif" font-size="14.00">BODY_FLAG_RESURRECT</text>
+<text text-anchor="middle" x="434.11" y="-64.95" font-family="Times,serif" font-size="14.00">iteration++</text>
+</g>
+</g>
+</svg>

doc/user/emulation/fig/parallel-tasks.svg

@@ -0,0 +1,176 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   width="117.73958mm"
+   height="26.458332mm"
+   viewBox="0 0 117.73958 26.458332"
+   version="1.1"
+   id="svg1"
+   inkscape:version="1.3 (0e150ed6c4, 2023-07-21)"
+   sodipodi:docname="parallel-tasks.svg"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg">
+  <sodipodi:namedview
+     id="namedview1"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:showpageshadow="2"
+     inkscape:pageopacity="0.0"
+     inkscape:pagecheckerboard="0"
+     inkscape:deskcolor="#d1d1d1"
+     inkscape:document-units="mm"
+     showgrid="true"
+     inkscape:zoom="1.4289282"
+     inkscape:cx="335.21629"
+     inkscape:cy="58.085495"
+     inkscape:window-width="1914"
+     inkscape:window-height="1025"
+     inkscape:window-x="0"
+     inkscape:window-y="24"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="layer1">
+    <inkscape:grid
+       id="grid1"
+       units="px"
+       originx="-9.9999997"
+       originy="-9.9999997"
+       spacingx="0.26458333"
+       spacingy="0.26458334"
+       empcolor="#3f3fff"
+       empopacity="0.25098039"
+       color="#3f3fff"
+       opacity="0.1254902"
+       empspacing="5"
+       dotted="false"
+       gridanglex="30"
+       gridanglez="30"
+       visible="true" />
+  </sodipodi:namedview>
+  <defs
+     id="defs1" />
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(-2.6458333,-2.6458333)">
+    <rect
+       style="fill:none;stroke:none;stroke-width:0.3;stroke-linecap:round"
+       id="rect9"
+       width="117.73958"
+       height="26.458332"
+       x="2.6458333"
+       y="2.6458333" />
+    <rect
+       style="fill:#f2f2f2;stroke:#000000;stroke-width:0.3;stroke-linecap:round"
+       id="rect1"
+       width="44.979168"
+       height="23.8125"
+       x="3.96875"
+       y="3.9687502" />
+    <text
+       xml:space="preserve"
+       style="font-size:4.23333px;line-height:1.25;font-family:monospace;-inkscape-font-specification:monospace;letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
+       x="6.6805291"
+       y="9.5720644"
+       id="text1"><tspan
+         sodipodi:role="line"
+         id="tspan1"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-family:'Droid Sans';-inkscape-font-specification:'Droid Sans';stroke-width:0.264583"
+         x="6.6805291"
+         y="9.5720644">Normal task</tspan></text>
+    <rect
+       style="fill:#d5ffd5;stroke:#000000;stroke-width:0.3;stroke-linecap:round"
+       id="rect3"
+       width="39.6875"
+       height="10.583333"
+       x="6.614583"
+       y="14.552083" />
+    <text
+       xml:space="preserve"
+       style="font-size:4.23333px;line-height:1.25;font-family:monospace;-inkscape-font-specification:monospace;letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
+       x="18.784389"
+       y="21.011637"
+       id="text3"><tspan
+         sodipodi:role="line"
+         id="tspan3"
+         style="stroke-width:0.264583"
+         x="18.784389"
+         y="21.011637">Body 1</tspan></text>
+    <rect
+       style="fill:#f2f2f2;stroke:#000000;stroke-width:0.3;stroke-linecap:round"
+       id="rect4"
+       width="67.46875"
+       height="23.8125"
+       x="51.59375"
+       y="3.9687502" />
+    <text
+       xml:space="preserve"
+       style="font-size:4.23333px;line-height:1.25;font-family:monospace;-inkscape-font-specification:monospace;letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
+       x="54.704952"
+       y="9.4792204"
+       id="text4"><tspan
+         sodipodi:role="line"
+         id="tspan4"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-family:'Droid Sans';-inkscape-font-specification:'Droid Sans';stroke-width:0.264583"
+         x="54.704952"
+         y="9.4792204">Parallel task</tspan></text>
+    <rect
+       style="fill:#d5ffd5;stroke:#000000;stroke-width:0.3;stroke-linecap:round"
+       id="rect6"
+       width="21.166662"
+       height="10.583335"
+       x="54.239586"
+       y="14.552083" />
+    <text
+       xml:space="preserve"
+       style="font-size:4.23333px;line-height:1.25;font-family:monospace;-inkscape-font-specification:monospace;letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
+       x="57.148972"
+       y="21.011637"
+       id="text6"><tspan
+         sodipodi:role="line"
+         id="tspan6"
+         style="stroke-width:0.264583"
+         x="57.148972"
+         y="21.011637">Body 1</tspan></text>
+    <rect
+       style="fill:#d5ffd5;stroke:#000000;stroke-width:0.3;stroke-linecap:round"
+       id="rect7"
+       width="21.166662"
+       height="10.583335"
+       x="78.052078"
+       y="14.552084" />
+    <text
+       xml:space="preserve"
+       style="font-size:4.23333px;line-height:1.25;font-family:monospace;-inkscape-font-specification:monospace;letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
+       x="80.997643"
+       y="21.011639"
+       id="text7"><tspan
+         sodipodi:role="line"
+         id="tspan7"
+         style="stroke-width:0.264583"
+         x="80.997643"
+         y="21.011639">Body 2</tspan></text>
+    <rect
+       style="fill:#d5ffd5;stroke:#000000;stroke-width:0.3;stroke-linecap:round"
+       id="rect8"
+       width="14.552085"
+       height="10.583334"
+       x="101.86458"
+       y="14.552084" />
+    <text
+       xml:space="preserve"
+       style="font-size:4.23333px;line-height:1.25;font-family:monospace;-inkscape-font-specification:monospace;letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
+       x="105.3207"
+       y="20.895952"
+       id="text8"><tspan
+         sodipodi:role="line"
+         id="tspan8"
+         style="stroke-width:0.264583"
+         x="105.3207"
+         y="20.895952">...</tspan></text>
+  </g>
+</svg>

doc/user/emulation/index.md

@@ -47,7 +47,7 @@ the following elements:
 
 - A single byte model identification (for example `O`).
 - A set of runtime events with that model identification (see the [list
-  of events](events)).
+  of events](events.md)).
 - Rules that determine which sequences of events are valid.
 - The emulation hooks that process each event and modify the state of
   the emulator.

doc/user/emulation/mpi.md

@@ -18,8 +18,8 @@ their functions.
 
 [mpi]: https://www.mpi-forum.org
 [mpi docs]: https://www.mpi-forum.org/docs
-[sonar]: https://pm.bsc.es/gitlab/ovni/sonar
-[sonar docs]: https://pm.bsc.es/gitlab/ovni/sonar/-/blob/main/README.md
+[sonar]: https://github.com/bsc-pm/sonar
+[sonar docs]: https://github.com/bsc-pm/sonar#readme
 
 Sonar requires an installation of the ovni library and an MPI library. Use the
 option `--with-ovni=prefix` when building Sonar to specify the ovni prefix. The

doc/user/emulation/nosv.md

@@ -1,9 +1,42 @@
 # nOS-V model
 
+The [nOS-V library][nosv] implements a user space runtime that can schedule
+tasks to run in multiple CPUs. The nOS-V library is instrumented to
+track the internal state of the runtime as well as emit information
+about the tasks that are running.
+
+[nosv]: https://github.com/bsc-pm/nos-v
+
+## Task model
+
 The nOS-V runtime is composed of tasks that can be scheduled to run in
 threads. Tasks can be paused and resumed, leaving the CPUs free to
-execute other tasks. The nOS-V model tracks the state of each task as
-well as the state of the runtime internal state.
+execute other tasks.
+
+In nOS-V, parallel tasks can also be scheduled multiple times and the
+same task may run concurrently in several CPUs. To model this scenario,
+we introduce the concept of *body*, which maps to each execution of the
+same task, with a unique body id.
+
+![Parallel tasks](fig/parallel-tasks.svg)
+
+A normal task only has one body, while a parallel task (created with
+`TASK_FLAG_PARALLEL`) can have more than one body. Each body holds the
+execution state, and can transition to different execution states
+following this state diagram:
+
+![Body model](fig/body-model.svg)
+
+Bodies begin in the Created state and transition to Running when they
+begin the execution. Bodies that can be paused (created with the flag
+`BODY_FLAG_PAUSE` can transition to the Paused state.
+
+Additionally, bodies can run multiple times if they are created with the
+`BODY_FLAG_RESURRECT`, and transition from Dead to Running. This
+transition is required to model the tasks that implement the taskiter in
+NODES, which will be submitted multiple times for execution reusing the
+same task id and body id. Every time a body runs again, the iteration
+number is increased.
 
 ## Task type colors
 
@@ -22,3 +55,52 @@ provides two desirable properties:
 For more details, see [this MR][1].
 
 [1]: https://pm.bsc.es/gitlab/rarias/ovni/-/merge_requests/27
+
+## Subsystem view
+
+The subsystem view provides a simplified view on what is the nOS-V
+runtime doing over time. The view follows the same rules described in
+the [subsystem view of Nanos6](nanos6.md/#subsystem_view).
+
+
+## Idle view
+
+The idle view shows the progress state of the running threads:
+*Progressing* and *Resting*. The *Progressing* state is shown when they
+are making useful progress and the *Resting* state when they are waiting
+for work. When workers start running, by definition, they begin in the
+Progressing state and there are some situations that make them
+transition to Resting:
+
+- When workers are waiting in the delegation lock after some spins or
+  when instructed to go to sleep.
+- When the server is trying to serve tasks, but there are no more tasks
+  available.
+
+They will go back to Progressing as soon as they receive work. The
+specific points at which they do so can be read in [nOS-V source
+code](https://gitlab.bsc.es/nos-v/nos-v) by looking at the
+`instr_worker_resting()` and `instr_worker_progressing()` trace points.
+
+This view is intended to detect parts of the execution time on which the
+workers don't have work, typically because the application doesn't have
+enough parallelism or the scheduler is unable to serve work fast enough.
+
+## Breakdown view
+
+The breakdown view displays a summary of what is happening in all CPUs
+by mixing in a single timeline the subsystem, idle and task type views.
+Specifically, it shows how many CPUs are resting as defined by the idle
+view, how many are inside a given task by showing the task type label,
+and how many are in a particular subsystem of the runtime.
+
+!!! Important
+
+    You must specify *ovni.level = 3* or higher in *nosv.toml* and pass
+    the *-b* option to ovniemu to generate the breakdown view.
+
+Notice that the vertical axis shows the **number**
+of CPUs in that state, not the physical CPUs like other views.
+Here is an example of the Heat mini-app:
+
+![Breakdown example](fig/breakdown-nosv.png)

doc/user/emulation/openmp.md

@@ -0,0 +1,264 @@
+# OpenMP model
+
+The [OpenMP programming model](https://www.openmp.org) is a widely used API and
+set of directives for parallel programming, allowing developers to write
+multi-threaded and multi-process applications more easily. In this document we
+refer to the
+[version 5.2 of the OpenMP specification](https://www.openmp.org/wp-content/uploads/OpenMP-API-Specification-5-2.pdf).
+
+The [LLVM OpenMP Runtime](https://openmp.llvm.org/design/Runtimes.html) provides
+an implementation of the OpenMP specification as a component of the LLVM
+compiler infrastructure. We have modified the LLVM OpenMP runtime (libomp) to run on top
+of the [nOS-V](https://gitlab.bsc.es/nos-v/nos-v) runtime as part of the
+[OmpSs-2 LLVM compiler](https://pm.bsc.es/llvm-ompss), named **libompv**.
+
+We have added instrumentation events to libompv designed to be enabled along
+the [nOS-V instrumentation](nosv.md). This document describes all the
+instrumentation features included in our modified libompv runtime to identify
+what is happening. This data is useful for both users and developers of the
+OpenMP runtime to analyze issues and undesired behaviors.
+
+!!! Note
+
+    Instrumenting libomp is planned but is not yet posible.
+    For now you must use libompv.
+
+## Enable the instrumentation
+
+To generate runtime traces, you will have to:
+
+1. **Build nOS-V with ovni support:** Refer to the
+  [nOS-V
+  documentation](https://github.com/bsc-pm/nos-v/blob/master/docs/user/tracing.md).
+  Typically you should use the `--with-ovni` option at configure time to specify
+  where ovni is installed.
+2. **Build libompv with ovni and nOS-V support:** Use the `PKG_CONFIG_PATH`
+  environment variable to specify the nOS-V and ovni installation 
+  when configuring CMake.
+3. **Enable the instrumentation in nOS-V at runtime:** Refer to the
+  [nOS-V documentation](https://github.com/bsc-pm/nos-v/blob/master/docs/user/tracing.md)
+  to find out how to enable the tracing at runtime. Typically you can just set 
+  `NOSV_CONFIG_OVERRIDE="instrumentation.version=ovni"`.
+4. **Enable the instrumentation of libompv at runtime:** Set the environment
+  variable `OMP_OVNI=1`.
+
+Next sections describe each of the views included for analysis.
+
+## Subsystem view
+
+![Subsystem view example](fig/openmp-subsystem.png)
+
+The view is complemented with the information of [nOS-V views](nosv.md),
+as libompv uses nOS-V tasks to run the workers.
+Subsystem illustrates the activities of each thread with different states:
+
+- **Work-distribution subsystem**: Related to work-distribution constructs,
+    [in Chapter 11][workdis].
+
+    - **Distribute**: Running a *Distribute* region.
+
+    - **Dynamic for chunk**: Running a chunk of a dynamic *for*, which often
+      involve running more than one iteration of the loop. See the
+      [limitations](#dynamic_for) below.
+
+    - **Dynamic for initialization**: Preparing a dynamic *for*.
+
+    - **Static for chunk**: Executing the assigned iterations of an static
+      *for*.
+
+    - **Single**: Running a *Single* region. All threads of the parallel region
+      participate.
+
+    - **Section**: Running a *Section* region. All threads of the parallel region
+      participate.
+
+- **Task subsystem**: Related to tasking constructs, [in Chapter 12][tasking].
+
+    - **Allocation**: Allocating the task descriptor.
+
+    - **Check deps**: Checking if the task has pending dependencies to be
+      fulfilled. When all dependencies are fulfilled the task will be scheduled.
+
+    - **Duplicating**: Duplicating the task descriptor in a taskloop.
+
+    - **Releasing deps**: Releasing dependencies at the end of a task. This
+      state is always present even if the task has no dependencies.
+
+    - **Running task**: Executing a task.
+
+    - **Running task if0**: Executing a task if0.
+
+    - **Scheduling**: Adding the task to the scheduler for execution.
+
+    - **Taskgroup**: Waiting in a *taskgroup* construct.
+
+    - **Taskwait**: Waiting in a *taskwait* construct.
+
+    - **Taskwait deps**: Trying to execute tasks until dependencies have been
+      fulfilled. This appears typically in a task if0 with dependencies or a
+      taskwait with deps.
+    
+    - **Taskyield**: Performing a *taskyield* construct.
+
+- **Critical subsystem**: Related to the *critical* Constuct, in [Section 15.2][critical].
+
+    - **Acquiring**: Waiting to acquire a *Critical* section.
+
+    - **Section**: Running the *Critical* section.
+
+    - **Releasing**: Waiting to release a *Critical* section.
+
+- **Barrier subsystem**: Related to barriers, in [Section 15.3][barrier].
+    **All barriers can try to execute tasks**.
+
+    - **Barrier: Fork**: Workers wait for a release signal from the master thread to
+      continue. The master can continue as soon as it signals the workers. It is
+      done at the beginning of a fork-join region.
+
+    - **Barrier: Join**: The master thread waits until all workers finish their work.
+      Workers can continue as soon as they signal the master. It is done at the
+      end of a fork-join region.
+  
+    - **Barrier: Plain**: Performing a plain barrier, which waits for a release
+      signal from the master thread to continue. It is done at the beginning of
+      a fork-join region, in the `__kmp_join_barrier()` function.
+
+    - **Barrier: Task**: Blocked in an additional tasking barrier *until all previous
+      tasks have been executed*. Only happens when executed with `KMP_TASKING=1`.
+
+- **Runtime subsystem**: Internal operations of the runtime.
+
+    - **Attached**: Present after the call to `nosv_attach()` and before
+      `nosv_detach()`. This state is a hack.
+
+    - **Fork call**: Preparing a parallel section using the fork-join model.
+      Only called from the master thread.
+
+    - **Init**: Initializing the libompv runtime.
+
+    - **Internal microtask**: Running a internal libompv function as a microtask.
+
+    - **User microtask**: Running user code as a microtask in a worker thread.
+
+    - **Worker main Loop**: Running the main loop, where the workers run the
+      fork barrier, run a microtask and perform a join barrier until there is no
+      more work.
+
+!!! Note
+
+    The generated HTML version of the OpenMP 5.2 specification has some parts
+    missing, so we link directly to the PDF file which may not work in some
+    browsers.
+
+[workdis]:  https://www.openmp.org/wp-content/uploads/OpenMP-API-Specification-5-2.pdf#chapter.11
+[tasking]:  https://www.openmp.org/wp-content/uploads/OpenMP-API-Specification-5-2.pdf#chapter.12
+[critical]: https://www.openmp.org/wp-content/uploads/OpenMP-API-Specification-5-2.pdf#section.15.2
+[barrier]:  https://www.openmp.org/wp-content/uploads/OpenMP-API-Specification-5-2.pdf#section.15.3
+
+## Label view
+
+The label view displays the text in the `label()` clause of OpenMP
+tasks and work distribution constructs (static and dynamic for, single
+and section). When the label is not provided, the source file and source
+line location is used instead.
+
+When nesting multiple tasks or work distribution constructs, only the
+innermost label is shown.
+
+Note that in this view, the numeric event value is a hash function of
+the type label, so two distinct tasks (declared in different parts of
+the code) with the same label will share the event value and have the
+same color.
+
+## Task ID view
+
+The task ID view represents the numeric ID of the OpenMP task that is
+currently running on each thread. The ID is a monotonically increasing
+identifier assigned on task creation. Lower IDs correspond to tasks
+created at an earlier point than higher IDs.
+
+## Limitations
+
+As the compiler generates the code that perform the calls to the libompv
+runtime, there are some parts of the execution that are complicated to
+instrument by just placing a pair of events to delimite a function.
+
+For those cases we use an approximation which is documented in the following
+subsections.
+
+### Dynamic for
+
+The generated code of a *dynamic for* has the following structure:
+
+```c
+__kmpc_dispatch_init_4(...);
+while (__kmpc_dispatch_next_4(...)) {
+    for (i = ...; i <= ...; i++) {
+        // User code ...
+    }
+}
+```
+
+The function `__kmpc_dispatch_next_4()` returns `true` if there are more
+chunks (group of iterations) to be executed by the thread, otherwise it returns
+`false`.
+
+Ideally we want to instrument each chunk with a pair of begin and end events.
+
+The problem with the instrumentation is that there is no easy way of determining
+if the call to `__kmpc_dispatch_next_4()` is processing the first chunk, just
+after `__kmpc_dispatch_init_4()`, or is coming from other chunks due to the
+while loop.
+
+Therefore, from the `__kmpc_dispatch_next_4()` alone, we cannot determine if we
+need to only emit a single "begin a new chunk" event or we need to emit the pair
+of events "finish the last chunk" and "begin a new one".
+
+So, as a workaround, we emit an event from the end of `__kmpc_dispatch_init_4()`
+starting a new chunk (which is fake), and then from `__kmpc_dispatch_next_4()` we
+always emit the "finish the last chunk" and "begin a new one" events (unless
+there are no more chunks, in which case we don't emit the "begin a new one"
+event).
+
+This will cause an spurious *Work-distribution: Dynamic for chunk* state at the
+beginning of each dynamic for, which should be very short and is not really a
+chunk.
+
+### Static for
+
+The generated code of an *static for* has the following structure:
+
+```c
+__kmpc_for_static_init_4(...);
+for (i = ...; i <= ...; i++) {
+    // User code ...
+}
+__kmpc_for_static_fini(...);
+```
+
+As this code is generated by the compiler we cannot easily add the begin/end
+pair of events to mark the *Work-distribution: Static for chunk* state.
+
+We assume that by placing the "begin processing a chunk" event at the end of
+`__kmpc_for_static_init_4()` and the "end processing the chunk" event at
+the beginning of `__kmpc_for_static_fini()` is equivalent to adding the
+events surrounding the for loop.
+
+### Task if0
+
+The generated code of an *if0 task* has the following structure:
+
+```c
+... = __kmpc_omp_task_alloc(...);
+__kmpc_omp_taskwait_deps_51(...); // If task has dependencies
+__kmpc_omp_task_begin_if0(...);
+// Call to the user code
+omp_task_entry_(...);
+__kmpc_omp_task_complete_if0(...);
+```
+
+Instead of injecting the begin and end events in the user code, we
+approximate it by placing the "begin if0 task" event at the end of the
+`__kmpc_omp_task_begin_if0` function and the "end if0 task" event at the
+beginning of `__kmpc_omp_task_complete_if0`. This state will be shown as 
+*Task: Running task if0*.

doc/user/emulation/ovni.md

@@ -60,4 +60,4 @@ will set all the channels to an error state.
 The emulator automatically switches the channels from one thread to
 another when a thread is switched from the CPU. So the different models
 don't need to worry about thread transitions. See the
-[channels](../channels) section for more information.
+[channels](../../dev/channels.md) section for more information.

doc/user/emulation/versions.md

@@ -0,0 +1,61 @@
+# Model versions
+
+Track changes in emulator model versions.
+
+!!! Note
+    These versions cover the [events defined](events.md) in the emulator models.
+    They are **not related to the versions of the libraries**.
+
+## Nanos6
+
+- nanos6 1.1.0:
+    - Deprecate running nested tasks (`6Tx`) without pausing the parent task
+      first (`6Tp`). For compatibility, the old behavior is still supported.
+- nanos6 1.0.0: Initial version
+
+## Nodes
+
+- nodes 1.0.0: Initial version
+
+## Kernel
+
+- kernel 1.0.0: Initial version
+
+## MPI
+
+- mpi 1.0.0: Initial version
+
+## Ovni
+
+- ovni 1.1.0
+    - Add support for mark events `OM[`, `OM]` and `OM=`
+- ovni 1.0.0: Initial version
+
+## OpenMP
+
+- openmp 1.2.0:
+    - Add support for labels and task ID views
+- openmp 1.1.0: Initial version
+
+## TAMPI
+
+- tampi 1.0.0: Initial version
+
+## nOS-V
+
+- nosv 2.4.0
+    - Add support for `nosv_cond_wait`, `nosv_cond_signal` and `nosv_cond_broadcast` events VA{oOgGkK}.
+- nosv 2.3.0
+    - Add `nosv.can_breakdown` attribute to metadata for breakdown checks.
+- nosv 2.2.0
+    - Add support for progress events `VP{pra}`.
+- nosv 2.1.0
+    - Add support for `nosv_mutex_lock`, `nosv_mutex_trylock` and `nosv_mutex_unlock` events `VA{lLtTuU}`.
+    - Add support for `nosv_barrier_wait` event `VA{bB}`.
+- nosv 2.0.0
+    - Add support for parallel tasks, adding a new `bodyid` argument in `VT*` events.
+    - Remove support for old attach events `VH{aA}`.
+- nosv 1.1.0
+    - Ignore old attach events `VH{aA}`.
+    - Add new API attach `VA{aA}` and detach `VA{eE}` events.
+- nosv 1.0.0: Initial version.

doc/user/runtime/distributed.md

@@ -24,9 +24,11 @@ If you are using SLURM, you may want to use something like:
 	you must invoke srun twice.
 
 By default, it will generate the `ovni/clock-offsets.txt` file, with the
-relative offsets to the rank 0 of MPI. The emulator will automatically pick the
-offsets when processing the trace. Use the ovnisync `-o` option to select a
-different output path (see the `-c` option in ovniemu to load the file).
+relative offsets to the rank 0 of MPI. If the `OVNI_TRACEDIR` environment
+variable is defined, the default file is `$OVNI_TRACEDIR/clock-offsets.txt`.
+The emulator will automatically pick the offsets when processing the trace.
+Use the ovnisync `-o` option to select a different output path (see the `-c`
+option in ovniemu to load the file).
 
 Here is an example table with three nodes, all units are in nanoseconds. The
 standard deviation is less than 1 us:

doc/user/runtime/env.md

@@ -0,0 +1,56 @@
+# Environment variables
+
+Some environment variables can be used to adjust settings during the execution
+of libovni, they all begin with the `OVNI_` prefix. Be sure that all threads of
+the same node use the same environment variables.
+
+## OVNI_TMPDIR
+
+During the execution of your program, a per-thread buffer is kept where the new
+events are being recorded. When this buffer is full, it is written to disk and
+emptied, an operation known as flush. This may take a while depending on the
+underlying filesystem.
+
+Keep in mind that the thread will be blocked until the flush ends, so if your
+filesystem is slow it would interrupt the execution of your program for a long
+time. It is advisable to use the fastest filesystem available (see the tmpfs(5)
+and df(1) manual pages).
+
+You can select a temporary trace directory where the buffers will be flushed
+during the execution by setting the environment variable `OVNI_TMPDIR`. The last
+directory will be created if doesn't exist. In that case, as soon as a process
+calls `ovni_proc_fini()`, the traces of all its threads will be moved to the
+final directory at `$PWD/ovni`. Example:
+
+	OVNI_TMPDIR=$(mktemp -u /dev/shm/ovni.XXXXXX) srun ./your-app
+
+To test the different filesystem speeds, you can use hyperfine and dd. Take a
+closer look at the max time:
+
+```
+$ hyperfine 'dd if=/dev/zero of=/gpfs/projects/bsc15/bsc15557/kk bs=2M count=10'
+Benchmark 1: dd if=/dev/zero of=/gpfs/projects/bsc15/bsc15557/kk bs=2M count=10
+  Time (mean ± σ):      71.7 ms ± 130.4 ms    [User: 0.8 ms, System: 10.2 ms]
+  Range (min … max):    14.7 ms … 1113.2 ms    162 runs
+ 
+  Warning: Statistical outliers were detected. Consider re-running this
+  benchmark on a quiet PC without any interferences from other programs. It
+  might help to use the '--warmup' or '--prepare' options.
+
+$ hyperfine 'dd if=/dev/zero of=/tmp/kk bs=2M count=10'
+Benchmark 1: dd if=/dev/zero of=/tmp/kk bs=2M count=10
+  Time (mean ± σ):      56.2 ms ±   5.7 ms    [User: 0.6 ms, System: 14.8 ms]
+  Range (min … max):    45.8 ms …  77.8 ms    63 runs
+ 
+$ hyperfine 'dd if=/dev/zero of=/dev/shm/kk bs=2M count=10'
+Benchmark 1: dd if=/dev/zero of=/dev/shm/kk bs=2M count=10
+  Time (mean ± σ):      11.4 ms ±   0.4 ms    [User: 0.5 ms, System: 11.1 ms]
+  Range (min … max):     9.7 ms …  12.5 ms    269 runs
+```
+
+## OVNI_TRACEDIR
+
+By default, the runtime trace will be placed in the `ovni` directory, inside the
+working directory. You can specify a different location to place the trace by
+setting the `OVNI_TRACEDIR` environment variable. It accepts a relative or
+absolute path, which will be created if it doesn't exist.

doc/user/runtime/index.md

@@ -0,0 +1,123 @@
+# Introduction
+
+To use *libovni* to instrument a program follow the next instructions
+carefully or you may end up with an incomplete trace that is rejected at
+emulation.
+
+You can also generate a valid trace from your own software or hardware
+directly following the [trace specification](trace_spec.md).
+
+## Initialization
+
+To initialize libovni follow these steps in all threads:
+
+1. **Check the version**. Call `ovni_version_check()` once before calling any
+   ovni function. It can be called multiple times from any thread, but only one
+   is required.
+
+2. **Init the process**. Call `ovni_proc_init()` to initialize the process. It
+   can only be called **once per process** and it must be called before the
+   thread is initialized.
+
+3. **Init the thread**. Call `ovni_thread_init()` to initialize the thread.
+   Multiple attempts to initialize the same thread are ignored with a warning.
+   Must be called by all threads.
+
+The `ovni_proc_init()` arguments are as follows:
+
+```c
+void ovni_proc_init(int app, const char *loom, int pid);
+```
+
+The `app` defines the "appid" of the program, which must be a number >0. This is
+useful to run multiple processes some of which run the same "app", so you can
+tell which one is which. The `loom` argument defines the
+[loom](../concepts/part-model.md#loom) name and maps the process to that
+loom. It must be composed of the host name, a dot and a suffix. The PID is the
+one obtained by `getpid(2)`.
+
+The `ovni_thread_init()` function only accepts one argument, the TID as returned
+by `gettid(2)`.
+
+## Setup metadata
+
+Once the process and thread are initialized, you can begin adding metadata to
+the thread stream.
+
+1. **Require models**. Call `ovni_thread_require()` with the required model
+   version before emitting events for a given model. Only required once from a
+   thread in a given trace. The `ovni` model is implicitly required when calling
+   `ovni_thread_init()`, so there is no need to add it again.
+
+2. **Emit loom CPUs**. Call `ovni_add_cpu()` to register each CPU in the loom. It can
+   be done from a single thread or multiple threads, in the latter the list of
+   CPUs is merged.
+
+3. **Set the rank**. If you use MPI, call `ovni_proc_set_rank()` to register the
+   rank and number of ranks of the current execution. Only once per process.
+
+When emitting the CPUs with:
+
+```c
+void ovni_add_cpu(int index, int phyid);
+```
+
+The `index` will be used to identify the CPU in the loom and goes from 0 to N -
+1, where N is the number of CPUs in the loom. It must match the index that is
+used in affinity events when a thread switches to another CPU. The `phyid` is
+only displayed in Paraver and is usually the same as the index, but it can be
+different if there are multiple looms per node.
+
+## Start the execution
+
+The current thread must switch to the "Running" state before any event can be
+processed by the emulator. Do so by emitting a [`OHx`
+event](../emulation/events.md#OHx) in the stream with the appropriate payload:
+
+```c
+static void thread_execute(int32_t cpu, int32_t ctid, uint64_t tag)
+{
+    struct ovni_ev ev = {0};
+    ovni_ev_set_clock(&ev, ovni_clock_now());
+    ovni_ev_set_mcv(&ev, "OHx");
+    ovni_payload_add(&ev, (uint8_t *) &cpu, sizeof(cpu));
+    ovni_payload_add(&ev, (uint8_t *) &ctid, sizeof(ctid));
+    ovni_payload_add(&ev, (uint8_t *) &tag, sizeof(tag));
+    ovni_ev_emit(&ev);
+}
+```
+
+The `cpu` is the logical index (not the physical ID) of the loom CPU at which
+this thread will begin the execution. Use -1 if it is not known. The `ctid` and
+`tag` allow you to track the exact point at which a given thread was created and
+by which thread but they are not relevant for the first thread, so they can be
+set to -1.
+
+## Emit events
+
+After this point you can emit any other event from this thread. Use the
+`ovni_ev_*` set of functions to create and emit events. Notice that all events
+refer to the current thread that emits them.
+
+If you need to store metadata information, use the `ovni_attr_*` set of
+functions. The metadata is stored in disk by `ovni_attr_flush()` and when the
+thread is freed by `ovni_thread_free()`.
+
+Attempting to emit events or writing metadata without having a thread
+initialized will cause your program to abort.
+
+## Finishing the execution
+
+To finalize the execution **every thread** must perform the following steps,
+otherwise the trace **will be rejected**.
+
+1. **End the current thread**. Emit a [`OHe` event](../emulation/events.md#OHe) to inform the current thread ends.
+2. **Flush the buffer**. Call `ovni_flush()` to be sure all events are written
+   to disk.
+3. **Free the thread**. Call `ovni_thread_free()` to complete the stream and
+   free the memory used by the buffer.
+4. **Finish the process**. If this is the last thread, call `ovni_proc_fini()`
+   to set the process state to finished.
+
+If a thread fails to perform these steps, the complete trace will be rejected by
+the emulator as it cannot guarantee it to be consistent.

doc/user/runtime/mark.md

@@ -0,0 +1,218 @@
+# Mark events
+
+The mark API allows you to add arbitrary events in a trace to mark regions of
+interest while debugging or developing a new program or library. The events are
+processed by the emulator to generate a timeline.
+
+## Usage in runtime
+
+Follow these steps to correctly use the API. Most problems will be detected in
+emulation and cause a panic if you are not careful.
+
+### Create a mark type
+
+You can create up to 100 types of marks, each will be shown in its own Paraver
+timeline. To create a new type, use the following call:
+
+```c
+void ovni_mark_type(int32_t type, long flags, const char *title);
+```
+
+The numeric value `type` must be a number between 0 and 99 (both included). The
+`title` will be used to give a name to the Paraver timeline.
+
+The default with flags set to zero, is to create a channel that can hold a
+single value only (see [channels](../../dev/channels.md)). To use a stack of
+values add the `OVNI_MARK_STACK` value to the flags.
+
+Only one thread among all nodes needs to define a type to be available globally,
+but the same type can be defined from multiple threads, as long as the same
+flags and title argument are used. The idea is to avoid having to check if the
+type was already defined or not.
+
+### Define labels (optional)
+
+The values that are written to the channel can have labels to display in the
+Paraver timeline. The labels are optional, if not given the numeric value will
+be shown in Paraver.
+
+Use the following call to register a label for a value in a given type.
+
+```c
+void ovni_mark_label(int32_t type, int64_t value, const char *label);
+```
+
+A value can only have at most a single label associated. Multiple threads can
+call the `ovni_mark_label()` with the same type and value as long as they use
+the same label. New labels for the same type can be associated from different
+threads, as long as the values are different.
+
+All value and label pairs are combined from all threads and will be available in
+the Paraver view for each type.
+
+### Emit events
+
+All mark channels begin with the default value *null*, which is not shown in
+Paraver and will be displayed as the usual empty space. The value of the channel
+can be changed over time with the following functions.
+
+!!! warning
+
+    The value 0 is forbidden, as it is used by Paraver to represent the
+    "empty" state.
+
+If you have used a single channel (without `OVNI_MARK_STACK`), then you must use
+the following call to emit events at runtime:
+
+```c
+void ovni_mark_set(int32_t type, int64_t value);
+```
+
+It will update the value of the channel to the given `value`.
+
+If you have used a stack channel (with `OVNI_MARK_STACK`), then you must use the
+push/pop set of calls:
+
+```c
+void ovni_mark_push(int32_t type, int64_t value);
+void ovni_mark_pop(int32_t type, int64_t value);
+```
+
+The value in the pop call must match the previous pushed value.
+
+<details>
+<summary>Example OmpSs-2 program</summary>
+<br>
+<div style="padding-left: 1em; border-left: 3px solid #ddd">
+<p>
+Here is a dummy program showing how to use the mark API with an OmpSs-2 program.
+Notice that there is no initialization of the current thread or process, as it
+already occurs inside the OmpSs-2 runtime before reaching the main.
+
+```c
+/* Build with:
+ *     $ clang -fompss-2 -lovni dummy.c -o dummy
+ * Enable instrumentation in nanos6:
+ *     $ echo 'version.instrument = "ovni"' > nanos6.toml
+ * Run:
+ *     $ ./dummy
+ * Emulate:
+ *     $ ovniemu ovni
+ * View timeline:
+ *     $ wxparaver ovni/cpu.prv ovni/cfg/cpu/ovni/mark.cfg
+ */
+#include <ovni.h>
+#include <unistd.h>
+
+enum { INDEX = 0, RUN = 1 };
+
+static void process(int run, int i)
+{
+    ovni_mark_push(RUN, run + 1);
+    ovni_mark_push(INDEX, i + 1);
+    usleep(10000); // Dummy operation for 10 ms
+    ovni_mark_pop(INDEX, i + 1);
+    ovni_mark_pop(RUN, run + 1);
+}
+
+int main(void)
+{
+    ovni_mark_type(INDEX, OVNI_MARK_STACK, "Index");
+    ovni_mark_type(RUN, OVNI_MARK_STACK, "Run");
+
+    for (int run = 0; run < 10; run++) {
+        for (int i = 0; i < 50; i++) {
+            #pragma oss task
+            process(run, i);
+        }
+    }
+
+    #pragma oss taskwait
+
+    return 0;
+}
+```
+
+<!-- Images don't seem to work via markdown -->
+<p>Here is the resulting timeline loaded in Paraver with the gradient color
+configuration, showing the first mark type (the index):
+<img style="margin-top: 1em" alt="" src="../fig/mark.png"></p>
+
+</div>
+</details>
+<p></p>
+
+## Usage in Paraver
+
+Each thread holds a channel for each mark type that you have defined. The
+information of the mark channels is propagated to the Paraver timeline in 
+Thread and CPU views.
+
+When a thread is not *running*, the value of the mark channels is not shown in
+Paraver. In the case of the CPU timeline, only the values of the running thread are
+shown. If there are no running threads, nothing is shown.
+
+Follow the next steps to create a configuration to suit your needs. You only
+need to do it once, then you can save the configuration file and reuse it for
+future traces.
+
+### Filtering the type
+
+To see a mark type, you will have to create a Paraver configuration that matches
+the type that you have created. The mark `type` value gets converted into a PRV
+type by adding 100 (as values from 0 to 99 are reserved).
+
+You can use the `cpu/ovni/mark.cfg` and `thread/ovni/mark.cfg` configurations as
+a starting point to create your own.
+
+Go to "Window Properties" (the second button under "Files & Window Properties")
+and then go to Filter > Events > Event type. Set Function to `=` and click the
+Types value to display the `[...]` button, which will allow you to choose which
+type to display.
+
+In the "Events Selection" window, ensure that only one type is selected, and the
+"Values" panel shows Function "All", to see all values for the selected type.
+
+### Setting the title
+
+In the "Window Properties" adjust the Name so it reflects what you are seeing.
+This will be shown in the saved images, so it is good to use a proper
+description.
+
+### Configure the display method
+
+By default, the timeline will display the values as "Code color". To switch to a
+gradient or other methods, left-click in the timeline and go to "Paint As" and
+select "Gradient" (or others).
+
+You may also want to adjust the "Drawmode" which determines what happens when
+there are multiple values under a given pixel. This is specially important when
+you are viewing the trace with a large time range, before zooming into a given
+region.
+
+By default, the "Random not zero" mode is selected, which will select a
+random value from the ones under each pixel, disregarding the occurrences of each
+value. This mode will give importance to rare values, so it is usually a safe
+starting point. The "Last" mode will show the last value in that pixel, which is
+more or less fair, but will often hide rare values.
+
+To change in both horizontal (Time) and in vertical (Objects) directions, go to:
+left click on timeline > Drawmode > Both > Last.
+
+### Ensure the range is good
+
+Paraver will only display values in the timeline that are in the Semantic 
+range. If you see a red triangle in the lower left corner then there are values
+outside the range that are not being displayed. You can click on this button to
+expand the range to cover all values in the current view.
+
+The opposite may also happen, where the range is too big for the current values.
+You can also click on the same spot (even if the triangle is not shown) to
+shrink the range to cover the values in the view, or go to the Window Properties
+and modify the "Semantic Minimum" and "Semantic Maximum" values manually.
+
+### Save the configuration
+
+Once you finish configuring the timeline, save the configuration by
+left-clicking the view and then "Save > Configuration...". You can use this
+configuration in future traces to avoid doing these steps again.

doc/user/runtime/trace_spec.md

@@ -1,88 +1,149 @@
-# Trace specification
+# Trace specification v3
 
 !!! Important
 
 	This document refers to the trace specification for
-	the version 1
+	the version 3
 
-The ovni instrumentation library stores the information collected in a
-trace following the specification of this document.
+The ovni instrumentation library libovni stores the information
+collected in a runtime trace following the specification of this document.
+
+## Structure
+
+An ovni runtime trace (or simply, a trace) is composed of one or more
+[streams](../concepts/trace.md#stream), which are directories containing
+two mandatory files:
+
+- `stream.json` the stream metadata in JSON format.
+- `stream.obs` the binary stream with events.
+
+Each stream is assigned to a single *part* in the [part
+model](../concepts/part-model.md), usually assigned to a given thread.
+
+There are no imposed rules on how to organize the several streams into
+directories, but libovni uses the following approach for thread streams:
 
 The complete trace is stored in a top-level directory named `ovni`.
-Inside this directory you will find the loom directories with the prefix
-`loom.`. The name of the loom is built from the `loom` parameter of
-`ovni_proc_init()`, prefixing it with `loom.`.
+Inside this directory you will find the loom directories. The name of
+the loom directory is built from the `loom` parameter of `ovni_proc_init()`,
+prefixing it with `loom.`.
 
 Each loom directory contains one directory per process of that loom. The
 name is composed of the `proc.` prefix and the PID of the process
 specified in the `pid` argument to `ovni_proc_init()`.
 
-Each process directory contains:
-
-- The metadata file `metadata.json`.
-- The thread streams like `thread.123.obs`.
-
-## Process metadata
-
-The metadata file contains important information about the trace that is
-invariant during the complete execution, and generally is required to be
-available prior to processing the events in the trace.
-
-The metadata is stored in the JSON file `metadata.json` inside each
-process directory and contains the following keys:
-
-- `version`: a number specifying the version of the metadata format.
-- `model_version`: a string with the version of each model supported by
-  the emulator.
-- `app_id`: the application ID, used to distinguish between applications
-  running on the same loom.
-- `rank`: the rank of the MPI process (optional).
-- `nranks`: number of total MPI processes (optional).
-- `cpus`: the array of $`N_c`$ CPUs available in the loom. Only one
-  process in the loom must contain this mandatory key. Each element is a
-  dictionary with the keys:
-  - `index`: containing the logical CPU index from 0 to $`N_c - 1`$.
-  - `phyid`: the number of the CPU as given by the operating system
-    (which can exceed $`N_c`$).
-
-Here is an example of the `metadata.json` file:
+Inside each process there is one directory for each thread, composed by
+the `thread.` prefix and the TID, which are the streams. The files
+`stream.json` and `stream.obs` reside inside. Example:
 
 ```
+ovni/loom.mio.nosv-u1000/proc.89719/thread.89719/stream.json
+ovni/loom.mio.nosv-u1000/proc.89719/thread.89719/stream.obs
+```
+
+This structure prevents collisions among threads with the same TID among nodes,
+while allowing dumping events from a single thread, process or loom with
+ovnidump.
+
+## Stream metadata
+
+The `stream.json` metadata file contains information about the part that
+the stream is assigned to. This is generally used to determine the
+hierarchy of the part model.
+
+The JSON must be an object (dictionary) with the following mandatory
+keys:
+
+- `version`: a number specifying the version of the metadata format.
+  Must have the value 3 for this version.
+
+The rest of information is stored for each model.
+
+In particular, the `ovni` model enforces the use of:
+
+- `ovni.part`: the type of part this stream is assigned to, usually
+  `thread`.
+- `ovni.require`: a dictionary of model name and version which will
+  determine which models are enabled at emulation and the required
+  version.
+- `ovni.finished`: must be 1 to ensure the stream is complete (mandatory
+  in all streams).
+
+### Thread stream metadata
+
+For `thread` streams, the following attributes are used.
+
+- `ovni.tid`: the TID of the thread (mandatory, per-thread).
+- `ovni.pid`: the PID of the process that the thread belongs to (mandatory, per-thread).
+- `ovni.app_id`: the application ID of the process (optional, per-process).
+- `ovni.rank`: the rank of the MPI process (optional, per-process).
+- `ovni.nranks`: number of total MPI processes (optional, per-process).
+- `ovni.loom`: the name of the loom that the process belongs to (mandatory, per-process).
+- `ovni.loom_cpus`: the array of N CPUs available in the loom
+  (mandatory, per-loom). Each element is a dictionary with the keys:
+    - `index`: containing the logical CPU index from 0 to N - 1.
+    - `phyid`: the number of the CPU as given by the operating system
+      (which can exceed N).
+
+Notice that some attributes don't need to be present in all thread
+streams. For example, per-process requires that at least one thread
+contains the attribute for each process. Similarly, per-loom requires
+that at least one thread of the loom emits the attribute.
+
+The final attribute value will be computed by merging all the values from the
+children metadata. Simple values like numbers or strings must match exactly if
+they appear duplicated, arrays are appended.
+
+Other attributes can be used for other models.
+
+Here is an example of the `stream.json` file for a thread of a nOS-V
+program:
+
+```json
 {
-    "version": 1,
-    "model_version": "O1 V1 T1 M1 D1 K1",
-    "app_id": 1,
-    "rank": 0,
-    "nranks": 4,
-    "cpus": [
-        {
-            "index": 0,
-            "phyid": 0
+    "version": 3,
+    "ovni": {
+        "lib": {
+            "version": "1.10.0",
+            "commit": "dirty"
         },
-        {
-            "index": 1,
-            "phyid": 1
+        "part": "thread",
+        "tid": 89719,
+        "pid": 89719,
+        "loom": "mio.nosv-u1000",
+        "app_id": 1,
+        "require": {
+            "ovni": "1.1.0",
+            "nosv": "2.3.0"
         },
-        {
-            "index": 2,
-            "phyid": 2
-        },
-        {
-            "index": 3,
-            "phyid": 3
-        }
-    ]
+        "loom_cpus": [
+            { "index": 0, "phyid": 0 },
+            { "index": 1, "phyid": 1 },
+            { "index": 2, "phyid": 2 },
+            { "index": 3, "phyid": 3 }
+        ],
+        "finished": 1
+    },
+    "nosv": {
+        "can_breakdown": false,
+        "lib_version": "2.3.1"
+    }
 }
 ```
 
-## Thread streams
+## Binary stream
 
-Streams are a binary files that contains a succession of events with
-monotonically increasing clock values. Streams have a small header and
-the variable size events just after the header.
+!!! Important
+
+	Binary streams have version 1
+
+A binary stream is a binary file named `stream.obs` that contains a
+succession of events with monotonically increasing clock values. They
+have a small header and the variable size events just after the header.
 
 The header contains the magic 4 bytes of "ovni" and a version number of
-4 bytes too. Here is a figure of the data stored in disk:
+4 bytes too. Here is a figure of the data stored in disk on a little
+endian machine:
 
 ![Stream](fig/stream.svg)
 
@@ -106,7 +167,7 @@ payload:
 - Normal events: with a payload up to 16 bytes
 - Jumbo events: with a payload up to $`2^{32}`$ bytes
 
-## Normal events
+### Normal events
 
 The normal events are composed of:
 
@@ -139,7 +200,7 @@ In the following figure you can see each field annotated:
 
 ![Normal event with payload content](fig/event-normal-payload.svg)
 
-## Jumbo events
+### Jumbo events
 
 The jumbo events are just like normal events but they can hold large
 data. The size of the jumbo data is stored as a 32 bits integer as a
@@ -164,10 +225,10 @@ In the following figure you can see each field annotated:
 
 ![Jumbo event](fig/event-jumbo.svg)
 
-## Design considerations
+### Design considerations
 
-The stream format has been designed to be very simple, so writing a
-parser library would take no more than 2 days for a single developer.
+The binary stream format has been designed to be very simple, so writing
+a parser library would take no more than 2 days for a single developer.
 
 The size of the events has been designed to be small, with 12 bytes per
 event when no payload is used.
@@ -200,11 +261,7 @@ raw stream in binary, as the MCV codes can be read as ASCII characters:
 This allows a human to detect signs of corruption by visually inspecting
 the streams.
 
-## Limitations
+### Limitations
 
 The streams are designed to be read only forward, as they only contain
 the size of each event in the header.
-
-Currently, we only support using the threads as sources of events, using
-one stream per thread. However, adding support for more streams from
-multiple sources is planned for the future.

doc/user/runtime/tracing.md

@@ -1,83 +0,0 @@
-# Tracing a new program
-
-Read carefully this document before using libovni to instrument a new
-component. There are a few rules you must follow to ensure the runtime
-trace is correct.
-
-## Trace processes and threads
-
-- Call `ovni_version_check()` once before calling any ovni function.
-
-- Call `ovni_proc_init()` when a new process begins the execution.
-
-- Call `ovni_thread_init()` when a new thread begins the execution
-(including the main process thread). Call `ovni_flush()` and
-`ovni_thread_free()` when it finishes (in that order).
-
-- Call `ovni_proc_fini()` when a process ends, after all threads have
-finished.
-
-You can use `ovni_ev_emit()` to record a new event. If you need more
-than 16 bytes of payload, use `ovni_ev_jumbo_emit()`. See the [trace
-specification](../trace_spec) for more details.
-
-Compile and link with libovni. When you run your program, a new
-directory ovni will be created in the current directory `$PWD/ovni`
-which contains the execution trace.
-
-## Rules
-
-Follow these rules to avoid losing events:
-
-1. No event may be emitted until the process is initialized with
-`ovni_proc_init()` and the thread with `ovni_thread_init()`.
-
-2. When a thread ends the execution, it must call `ovni_flush()` to write the
-events in the buffer to disk.
-
-3. All threads must have flushed its buffers before calling `ovni_proc_fini()`.
-
-## Select a fast directory
-
-During the execution of your program, a per-thread buffer is kept where the new
-events are being recorded. When this buffer is full, it is written to disk and
-emptied, an operation known as flush. This may take a while depending on the
-underliying filesystem.
-
-Keep in mind that the thread will be blocked until the flush ends, so if your
-filesystem is slow it would interrupt the execution of your program for a long
-time. It is advisable to use the fastest filesystem available (see the tmpfs(5)
-and df(1) manual pages).
-
-You can select the trace directory where the buffers will be flushed during the
-execution by setting the environment variable `OVNI_TMPDIR`. The last directory
-will be created if doesn't exist. In that case, as soon as a process calls
-`ovni_proc_fini()`, the traces of all its threads will be moved to the final
-directory at `$PWD/ovni`. Example:
-
-	OVNI_TMPDIR=$(mktemp -u /dev/shm/ovni.XXXXXX) srun ./your-app
-
-To test the different filesystem speeds, you can use hyperfine and dd. Take a
-closer look at the max time:
-
-```
-$ hyperfine 'dd if=/dev/zero of=/gpfs/projects/bsc15/bsc15557/kk bs=2M count=10'
-Benchmark 1: dd if=/dev/zero of=/gpfs/projects/bsc15/bsc15557/kk bs=2M count=10
-  Time (mean ± σ):      71.7 ms ± 130.4 ms    [User: 0.8 ms, System: 10.2 ms]
-  Range (min … max):    14.7 ms … 1113.2 ms    162 runs
- 
-  Warning: Statistical outliers were detected. Consider re-running this
-  benchmark on a quiet PC without any interferences from other programs. It
-  might help to use the '--warmup' or '--prepare' options.
-
-$ hyperfine 'dd if=/dev/zero of=/tmp/kk bs=2M count=10'
-Benchmark 1: dd if=/dev/zero of=/tmp/kk bs=2M count=10
-  Time (mean ± σ):      56.2 ms ±   5.7 ms    [User: 0.6 ms, System: 14.8 ms]
-  Range (min … max):    45.8 ms …  77.8 ms    63 runs
- 
-$ hyperfine 'dd if=/dev/zero of=/dev/shm/kk bs=2M count=10'
-Benchmark 1: dd if=/dev/zero of=/dev/shm/kk bs=2M count=10
-  Time (mean ± σ):      11.4 ms ±   0.4 ms    [User: 0.5 ms, System: 11.1 ms]
-  Range (min … max):     9.7 ms …  12.5 ms    269 runs
-```
-

flake.lock

@@ -0,0 +1,45 @@
+{
+  "nodes": {
+    "bscpkgs": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1705310446,
+        "narHash": "sha256-PaPnkGotb2omIV6OsS72MGkqNN6Q/iHLlXQZ6S3vWOY=",
+        "ref": "refs/heads/master",
+        "rev": "3b21a32d835ff06741d5d59cd023ff2ae1ecb19f",
+        "revCount": 932,
+        "type": "git",
+        "url": "https://git.sr.ht/~rodarima/bscpkgs"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://git.sr.ht/~rodarima/bscpkgs"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1693663421,
+        "narHash": "sha256-ImMIlWE/idjcZAfxKK8sQA7A1Gi/O58u5/CJA+mxvl8=",
+        "path": "/nix/store/wl5m5xfayd69ycyspzyd4rilfgl6wmh0-source",
+        "rev": "e56990880811a451abd32515698c712788be5720",
+        "type": "path"
+      },
+      "original": {
+        "id": "nixpkgs",
+        "type": "indirect"
+      }
+    },
+    "root": {
+      "inputs": {
+        "bscpkgs": "bscpkgs",
+        "nixpkgs": "nixpkgs"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,175 @@
+{
+  inputs.nixpkgs.url = "nixpkgs";
+  inputs.bscpkgs.url = "git+https://git.sr.ht/~rodarima/bscpkgs";
+  inputs.bscpkgs.inputs.nixpkgs.follows = "nixpkgs";
+
+  nixConfig.bash-prompt = "\[nix-develop\]$ ";
+
+  outputs = { self, nixpkgs, bscpkgs }:
+  let
+    # Set to true to replace all libovni in all runtimes with the current
+    # source. Causes large rebuilds on changes of ovni.
+    useLocalOvni = false;
+
+    ovniOverlay = final: prev: {
+      nosv = prev.nosv.override {
+        useGit = true;
+        gitBranch = "master";
+        gitCommit = "3ca2f67993f85aa73c53f810ff12148189eae642";
+      };
+      nanos6 = prev.nanos6.override {
+        useGit = true;
+        gitBranch = "master";
+        gitCommit = "f39ea57c67a613d098050e2bb251116a021e91e5";
+      };
+      nodes = prev.nodes.override {
+        useGit = true;
+        gitBranch = "master";
+        gitCommit = "c97d7ca6f885500121a94c75df429c788e8d6cf8";
+      };
+      clangOmpss2Unwrapped = prev.clangOmpss2Unwrapped.override {
+        useGit = true;
+        gitBranch = "master";
+        gitCommit = "b7af30b36be3e7e90b33c5f01a3f7e3656df785f";
+      };
+      openmp = prev.openmp.overrideAttrs (old: {
+        # Newer version of LLVM OpenMP requires python3
+        nativeBuildInputs = (old.nativeBuildInputs or []) ++ [ final.python3 ];
+      });
+
+      # Use a fixed commit for libovni
+      ovniFixed = prev.ovni.override {
+        useGit = true;
+        gitBranch = "master";
+        gitCommit = "3bbfe0f0ecdf58e3f46ebafdf2540680f990b76b";
+      };
+      # Build with the current source
+      ovniLocal = prev.ovni.overrideAttrs (old: rec {
+        pname = "ovni-local";
+        version = if self ? shortRev then self.shortRev else "dirty";
+        src = self;
+        cmakeFlags = [ "-DOVNI_GIT_COMMIT=${version}" ];
+      });
+      # Select correct ovni for libovni
+      ovni = if (useLocalOvni) then final.ovniLocal else final.ovniFixed;
+    };
+    pkgs = import nixpkgs {
+      system = "x86_64-linux";
+      overlays = [
+        bscpkgs.bscOverlay
+        ovniOverlay
+      ];
+    };
+    compilerList = with pkgs; [
+      #gcc49Stdenv # broken
+      gcc6Stdenv
+      gcc7Stdenv
+      gcc8Stdenv
+      gcc9Stdenv
+      gcc10Stdenv
+      gcc11Stdenv
+      gcc12Stdenv
+      gcc13Stdenv
+    ];
+    lib = pkgs.lib;
+  in {
+    packages.x86_64-linux.ovniPackages = {
+      # Allow inspection of packages from the command line
+      inherit pkgs;
+    } // rec {
+      # Build with the current source
+      local = pkgs.ovniLocal;
+
+      # Build in Debug mode
+      debug = local.overrideAttrs (old: {
+        pname = "ovni-debug";
+        cmakeBuildType = "Debug";
+      });
+
+      # Without MPI
+      nompi = local.overrideAttrs (old: {
+        pname = "ovni-nompi";
+        buildInputs = lib.filter (x: x != pkgs.mpi ) old.buildInputs;
+        cmakeFlags = old.cmakeFlags ++ [ "-DUSE_MPI=OFF" ];
+      });
+
+      # Helper function to build with different compilers
+      genOvniCC = stdenv: (local.override {
+        stdenv = stdenv;
+      }).overrideAttrs (old: {
+        name = "ovni-gcc" + stdenv.cc.cc.version;
+        cmakeFlags = old.cmakeFlags ++ [
+          "-DCMAKE_INTERPROCEDURAL_OPTIMIZATION=OFF"
+        ];
+      });
+
+      # Test several gcc versions
+      compilers = let
+        drvs = map genOvniCC compilerList;
+      in pkgs.runCommand "ovni-compilers" { } ''
+        printf "%s\n" ${builtins.toString drvs} > $out
+      '';
+
+      # Enable RT tests
+      rt = (local.override {
+        # Provide the clang compiler as default
+        stdenv = pkgs.stdenvClangOmpss2;
+      }).overrideAttrs (old: {
+        pname = "ovni-rt";
+        # We need to be able to exit the chroot to run Nanos6 tests, as they
+        # require access to /sys for hwloc
+        __noChroot = true;
+        buildInputs = old.buildInputs ++ (with pkgs; [ pkg-config nosv nanos6 nodes openmpv ]);
+        cmakeFlags = old.cmakeFlags ++ [ "-DENABLE_ALL_TESTS=ON" ];
+        preConfigure = old.preConfigure or "" + ''
+          export NOSV_HOME="${pkgs.nosv}"
+          export NODES_HOME="${pkgs.nodes}"
+          export NANOS6_HOME="${pkgs.nanos6}"
+          export OPENMP_RUNTIME="libompv"
+        '';
+      });
+
+      # Build with ASAN and pass RT tests
+      asan = rt.overrideAttrs (old: {
+        pname = "ovni-asan";
+        cmakeFlags = old.cmakeFlags ++ [ "-DCMAKE_BUILD_TYPE=Asan" ];
+        # Ignore leaks in tests for now, only check memory errors
+        preCheck = old.preCheck + ''
+          export ASAN_OPTIONS=detect_leaks=0
+        '';
+      });
+
+      ubsan = rt.overrideAttrs (old: {
+        pname = "ovni-ubsan";
+        cmakeFlags = old.cmakeFlags ++ [ "-DCMAKE_BUILD_TYPE=Ubsan" ];
+      });
+
+      armv7 = (pkgs.pkgsCross.armv7l-hf-multiplatform.ovniLocal.overrideAttrs (old: {
+        pname = "ovni-armv7";
+        buildInputs = [];
+        nativeBuildInputs = [ pkgs.pkgsCross.armv7l-hf-multiplatform.buildPackages.cmake ];
+        cmakeFlags = old.cmakeFlags ++ [ "-DUSE_MPI=OFF" ];
+      })).overrideDerivation (old: {
+        doCheck = true;
+      });
+
+      aarch64 = (pkgs.pkgsCross.aarch64-multiplatform.ovniLocal.overrideAttrs (old: {
+        pname = "ovni-aarch64";
+        buildInputs = [];
+        nativeBuildInputs = [ pkgs.pkgsCross.aarch64-multiplatform.buildPackages.cmake ];
+        cmakeFlags = old.cmakeFlags ++ [ "-DUSE_MPI=OFF" ];
+      })).overrideDerivation (old: {
+        doCheck = true;
+      });
+
+      riscv64 = (pkgs.pkgsCross.riscv64.ovniLocal.overrideAttrs (old: {
+        pname = "ovni-riscv64";
+        buildInputs = [];
+        nativeBuildInputs = [ pkgs.pkgsCross.riscv64.buildPackages.cmake ];
+        cmakeFlags = old.cmakeFlags ++ [ "-DUSE_MPI=OFF" ];
+      })).overrideDerivation (old: {
+        doCheck = true;
+      });
+    };
+  };
+}

include/ovni.h.in

@@ -1,4 +1,4 @@
-/* Copyright (c) 2021-2023 Barcelona Supercomputing Center (BSC)
+/* Copyright (c) 2021-2024 Barcelona Supercomputing Center (BSC)
  * SPDX-License-Identifier: MIT */
 
 #ifndef OVNI_H
@@ -18,10 +18,7 @@ extern "C" {
 #include <string.h>
 #include <sys/types.h>
 
-/* Hardcode the JSON_Value to avoid a dependency with janson */
-typedef struct json_value_t JSON_Value;
-
-#define OVNI_METADATA_VERSION 1
+#define OVNI_METADATA_VERSION 3
 
 #define OVNI_TRACEDIR "ovni"
 #define OVNI_MAX_HOSTNAME 512
@@ -31,10 +28,12 @@ typedef struct json_value_t JSON_Value;
 
 #define OVNI_STREAM_MAGIC "ovni"
 #define OVNI_STREAM_VERSION 1
-#define OVNI_MODEL_VERSION "O1 V1 T1 M1 D1 K1 61"
 
 #define OVNI_STREAM_EXT ".obs"
 
+/* Version of the ovni model for events */
+#define OVNI_MODEL_VERSION "1.1.0"
+
 /* Follow https://semver.org rules for versioning */
 #define OVNI_LIB_VERSION "@PROJECT_VERSION@"
 #define OVNI_GIT_COMMIT  "@OVNI_GIT_COMMIT@"
@@ -89,45 +88,6 @@ struct __attribute__((__packed__)) ovni_stream_header {
 
 /* ----------------------- runtime ------------------------ */
 
-/* State of each thread on runtime */
-struct ovni_rthread {
-	/* Current thread id */
-	pid_t tid;
-
-	/* Stream trace file descriptor */
-	int streamfd;
-
-	int ready;
-
-	/* The number of bytes filled with events */
-	size_t evlen;
-
-	/* Buffer to write events */
-	uint8_t *evbuf;
-};
-
-/* State of each process on runtime */
-struct ovni_rproc {
-	/* Where the process trace is finally copied */
-	char procdir_final[PATH_MAX];
-
-	/* Where the process trace is flushed */
-	char procdir[PATH_MAX];
-
-	/* If needs to be moved at the end */
-	int move_to_final;
-
-	int app;
-	int pid;
-	char loom[OVNI_MAX_HOSTNAME];
-	int ncpus;
-	clockid_t clockid;
-
-	int ready;
-
-	JSON_Value *meta;
-};
-
 #define ovni_version_check() ovni_version_check_str(OVNI_LIB_VERSION)
 void ovni_version_check_str(const char *version);
 void ovni_version_get(const char **version, const char **commit);
@@ -141,6 +101,8 @@ void ovni_proc_fini(void);
 
 void ovni_thread_init(pid_t tid);
 
+void ovni_thread_require(const char *model, const char *version);
+
 void ovni_thread_free(void);
 
 int ovni_thread_isready(void);
@@ -171,6 +133,28 @@ void ovni_ev_jumbo_emit(struct ovni_ev *ev, const uint8_t *buf, uint32_t bufsize
 
 void ovni_flush(void);
 
+/* Attributes */
+int ovni_attr_has(const char *key);
+void ovni_attr_set_double(const char *key, double num);
+void ovni_attr_set_boolean(const char *key, int value);
+void ovni_attr_set_str(const char *key, const char *value);
+void ovni_attr_set_json(const char *key, const char *json);
+double ovni_attr_get_double(const char *key);
+int ovni_attr_get_boolean(const char *key);
+const char *ovni_attr_get_str(const char *key);
+char *ovni_attr_get_json(const char *key);
+void ovni_attr_flush(void);
+
+/* Mark */
+enum ovni_mark_flags {
+	OVNI_MARK_STACK = 1, /*< Use push/pop instead of set */
+};
+void ovni_mark_type(int32_t type, long flags, const char *title);
+void ovni_mark_label(int32_t type, int64_t value, const char *label);
+void ovni_mark_push(int32_t type, int64_t value);
+void ovni_mark_pop(int32_t type, int64_t value);
+void ovni_mark_set(int32_t type, int64_t value);
+
 #ifdef __cplusplus
 }
 #endif

mkdocs.yml

@@ -4,6 +4,9 @@ extra_css: [extra.css]
 theme:
   name: readthedocs
   sticky_navigation: false
+  highlightjs: true
+  hljs_languages:
+    - abnf
 extra_javascript: 
     - https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-AMS-MML_HTMLorMML
 markdown_extensions:
@@ -20,10 +23,14 @@ markdown_extensions:
 nav:
   - index.md
   - 'User guide':
-    - user/concepts.md
     - user/installation.md
+    - 'Concepts':
+      - user/concepts/part-model.md
+      - user/concepts/trace.md
     - 'Runtime':
-      - user/runtime/tracing.md
+      - user/runtime/index.md
+      - user/runtime/env.md
+      - user/runtime/mark.md
       - user/runtime/distributed.md
       - user/runtime/kernel.md
       - user/runtime/trace_spec.md
@@ -34,9 +41,13 @@ nav:
       - user/emulation/nanos6.md
       - user/emulation/tampi.md
       - user/emulation/mpi.md
+      - user/emulation/openmp.md
       - user/emulation/events.md
+      - user/emulation/versions.md
+    - CHANGELOG.md
   - 'Developer guide':
     - dev/index.md
+    - dev/events.md
     - dev/channels.md
     - dev/patchbay.md
     - dev/mux.md