Compare commits
No commits in common. "625161556a8a55086e05e7566b901ec8727c8fbb" and "ac12be8c129007cf1b25be0ffdabec4a4b8f05df" have entirely different histories.
625161556a
...
ac12be8c12
@ -1,8 +0,0 @@
|
||||
build:ci:
|
||||
stage: build
|
||||
tags:
|
||||
- nix
|
||||
script:
|
||||
- test/ci.sh
|
||||
rules:
|
||||
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
|
||||
58
flake.lock
58
flake.lock
@ -11,11 +11,11 @@
|
||||
"systems": "systems"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1712079060,
|
||||
"narHash": "sha256-/JdiT9t+zzjChc5qQiF+jhrVhRt8figYH29rZO7pFe4=",
|
||||
"lastModified": 1707830867,
|
||||
"narHash": "sha256-PAdwm5QqdlwIqGrfzzvzZubM+FXtilekQ/FA0cI49/o=",
|
||||
"owner": "ryantm",
|
||||
"repo": "agenix",
|
||||
"rev": "1381a759b205dff7a6818733118d02253340fd5e",
|
||||
"rev": "8cb01a0e717311680e0cbca06a76cbceba6f3ed6",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@ -26,19 +26,13 @@
|
||||
},
|
||||
"bscpkgs": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"jungle",
|
||||
"nixpkgs"
|
||||
]
|
||||
"nixpkgs": "nixpkgs"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1713974364,
|
||||
"narHash": "sha256-ilZTVWSaNP1ibhQIIRXE+q9Lj2XOH+F9W3Co4QyY1eU=",
|
||||
"ref": "refs/heads/master",
|
||||
"rev": "de89197a4a7b162db7df9d41c9d07759d87c5709",
|
||||
"revCount": 937,
|
||||
"type": "git",
|
||||
"url": "https://git.sr.ht/~rodarima/bscpkgs"
|
||||
"lastModified": 1712926972,
|
||||
"narHash": "sha256-0dKhCH3cPU8QSc8LLpJypj4FYGVbE5i2iRl6NnDhB6s=",
|
||||
"path": "/home/Computational/rarias/bscpkgs",
|
||||
"type": "path"
|
||||
},
|
||||
"original": {
|
||||
"type": "git",
|
||||
@ -94,29 +88,39 @@
|
||||
"inputs": {
|
||||
"agenix": "agenix",
|
||||
"bscpkgs": "bscpkgs",
|
||||
"nixpkgs": "nixpkgs"
|
||||
"nixpkgs": "nixpkgs_2"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1714044311,
|
||||
"narHash": "sha256-mEMtClaRZE45THl0Ukdhj5SVQzYSvRCRAdPshSlvkCQ=",
|
||||
"ref": "refs/heads/master",
|
||||
"rev": "d2adc3a6d3ab6a42cc0a1c99c2ff8681667409e0",
|
||||
"revCount": 235,
|
||||
"type": "git",
|
||||
"url": "https://git.sr.ht/~rodarima/jungle"
|
||||
"lastModified": 1710869393,
|
||||
"narHash": "sha256-Oi2lgWkYRG1SHImng09H2yQ06VS6nN7s6QPlWSPrTA4=",
|
||||
"path": "/home/Computational/rarias/jungle",
|
||||
"type": "path"
|
||||
},
|
||||
"original": {
|
||||
"type": "git",
|
||||
"url": "https://git.sr.ht/~rodarima/jungle"
|
||||
"path": "/home/Computational/rarias/jungle",
|
||||
"type": "path"
|
||||
}
|
||||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1713714899,
|
||||
"narHash": "sha256-+z/XjO3QJs5rLE5UOf015gdVauVRQd2vZtsFkaXBq2Y=",
|
||||
"lastModified": 1710272261,
|
||||
"narHash": "sha256-g0bDwXFmTE7uGDOs9HcJsfLFhH7fOsASbAuOzDC+fhQ=",
|
||||
"path": "/nix/store/k5l01g2zwhysjyl5zjvg5zxnj0lyxpp1-source",
|
||||
"rev": "0ad13a6833440b8e238947e47bea7f11071dc2b2",
|
||||
"type": "path"
|
||||
},
|
||||
"original": {
|
||||
"id": "nixpkgs",
|
||||
"type": "indirect"
|
||||
}
|
||||
},
|
||||
"nixpkgs_2": {
|
||||
"locked": {
|
||||
"lastModified": 1710272261,
|
||||
"narHash": "sha256-g0bDwXFmTE7uGDOs9HcJsfLFhH7fOsASbAuOzDC+fhQ=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "6143fc5eeb9c4f00163267708e26191d1e918932",
|
||||
"rev": "0ad13a6833440b8e238947e47bea7f11071dc2b2",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
||||
19
flake.nix
19
flake.nix
@ -2,23 +2,21 @@
|
||||
description = "bench6";
|
||||
nixConfig.bash-prompt = "\[nix-develop\]$ ";
|
||||
|
||||
inputs.jungle.url = "git+https://git.sr.ht/~rodarima/jungle";
|
||||
inputs.jungle.url = "path:/home/Computational/rarias/jungle";
|
||||
inputs.jungle.inputs.bscpkgs.url = "path:/home/Computational/rarias/bscpkgs";
|
||||
|
||||
outputs = { self, jungle, ... }:
|
||||
let
|
||||
targetMachine = jungle.outputs.nixosConfigurations.hut;
|
||||
pkgs = targetMachine.pkgs;
|
||||
in {
|
||||
packages.x86_64-linux = rec {
|
||||
default = bench6;
|
||||
bench6 = pkgs.stdenv.mkDerivation rec {
|
||||
packages.x86_64-linux.default = pkgs.stdenv.mkDerivation rec {
|
||||
pname = "bench6";
|
||||
version = if self ? shortRev then self.shortRev else "dirty";
|
||||
|
||||
src = self.outPath;
|
||||
|
||||
buildInputs = with pkgs; [
|
||||
bigotes
|
||||
cmake
|
||||
clangOmpss2
|
||||
nanos6
|
||||
@ -32,16 +30,5 @@
|
||||
hardeningDisable = [ "all" ];
|
||||
dontStrip = true;
|
||||
};
|
||||
|
||||
bench6Master = bench6.overrideAttrs (old: {
|
||||
src = builtins.fetchGit {
|
||||
url = "https://pm.bsc.es/gitlab/rarias/bench6.git";
|
||||
#ref = "master";
|
||||
# FIXME: Just for testing
|
||||
ref = "heat";
|
||||
rev = "466e50e511e11087f6e9298b3ac851fcab7d459c";
|
||||
};
|
||||
});
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
add_subdirectory(bench6)
|
||||
add_subdirectory(ompss2)
|
||||
add_subdirectory(heat)
|
||||
add_subdirectory(tools)
|
||||
|
||||
7
src/tools/CMakeLists.txt
Normal file
7
src/tools/CMakeLists.txt
Normal file
@ -0,0 +1,7 @@
|
||||
get_property(BENCH6_LIST GLOBAL PROPERTY bench6_list)
|
||||
configure_file(config.in.h config.h)
|
||||
include_directories(${CMAKE_CURRENT_BINARY_DIR})
|
||||
|
||||
add_executable(bigote runner.c)
|
||||
target_link_libraries(bigote PRIVATE m bench6_lib)
|
||||
install(TARGETS bigote RUNTIME DESTINATION bin)
|
||||
10
src/tools/config.in.h
Normal file
10
src/tools/config.in.h
Normal file
@ -0,0 +1,10 @@
|
||||
/* Copyright (c) 2023 Barcelona Supercomputing Center (BSC)
|
||||
* SPDX-License-Identifier: GPL-3.0-or-later */
|
||||
|
||||
#ifndef CONFIG_H
|
||||
#define CONFIG_H
|
||||
|
||||
/* Where the benchmark binaries are located */
|
||||
#define BENCH6_BENCHDIR "@CMAKE_INSTALL_LIBEXECDIR@"
|
||||
|
||||
#endif /* CONFIG_H */
|
||||
352
src/tools/runner.c
Normal file
352
src/tools/runner.c
Normal file
@ -0,0 +1,352 @@
|
||||
#include "bench6.h"
|
||||
#include "common.h"
|
||||
#include "config.h"
|
||||
#include <limits.h>
|
||||
#include <math.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/wait.h>
|
||||
#include <unistd.h>
|
||||
|
||||
static char *progname = "bench6";
|
||||
|
||||
struct sampling {
|
||||
long nmax;
|
||||
long nmin;
|
||||
long n;
|
||||
double *samples;
|
||||
double rsem;
|
||||
double last;
|
||||
double wall;
|
||||
double min_rsem;
|
||||
const char *name;
|
||||
double t0;
|
||||
double min_time;
|
||||
};
|
||||
|
||||
static int
|
||||
do_run(char *argv[], double *ptime)
|
||||
{
|
||||
int ret = 0;
|
||||
FILE *p = popen(argv[0], "r");
|
||||
|
||||
if (p == NULL) {
|
||||
err("popen failed:");
|
||||
return -1;
|
||||
}
|
||||
|
||||
char line[4096];
|
||||
if (fgets(line, 4096, p) == NULL) {
|
||||
err("missing stdout line");
|
||||
ret = -1;
|
||||
goto bad_close;
|
||||
}
|
||||
|
||||
char *nl = strchr(line, '\n');
|
||||
if (nl != NULL)
|
||||
*nl = '\0';
|
||||
|
||||
/* Clean status line */
|
||||
//fprintf(stderr, "%s\n", line);
|
||||
|
||||
double time;
|
||||
sscanf(line, "%le", &time);
|
||||
//printf("got %e\n", time);
|
||||
*ptime = time;
|
||||
|
||||
/* Drain the rest of the stdout */
|
||||
while (fgets(line, 4096, p) != NULL) {
|
||||
//fprintf(stderr, "%s", line);
|
||||
}
|
||||
|
||||
bad_close:
|
||||
pclose(p);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int
|
||||
cmp_double(const void *pa, const void *pb)
|
||||
{
|
||||
double a = *(const double *) pa;
|
||||
double b = *(const double *) pb;
|
||||
|
||||
if (a < b)
|
||||
return -1;
|
||||
else if (a > b)
|
||||
return 1;
|
||||
else
|
||||
return 0;
|
||||
}
|
||||
|
||||
//static void
|
||||
//resample(double *values, long n, double *out)
|
||||
//{
|
||||
// for (long i = 0; i < n; i++) {
|
||||
// /* FIXME: Not really uniform */
|
||||
// out[i] = values[rand() % n];
|
||||
// //printf("out[%ld] = %e\n", i, out[i]);
|
||||
// }
|
||||
//}
|
||||
//
|
||||
//static double
|
||||
//mad_bootstrap(double *values, long n)
|
||||
//{
|
||||
// long m = 1000;
|
||||
//
|
||||
// double *r = calloc(n, sizeof(double));
|
||||
// if (r == NULL) {
|
||||
// perror("calloc failed");
|
||||
// exit(1);
|
||||
// }
|
||||
//
|
||||
// double *absdev = calloc(n, sizeof(double));
|
||||
// if (absdev == NULL) {
|
||||
// perror("calloc failed");
|
||||
// exit(1);
|
||||
// }
|
||||
//
|
||||
// double *mad = calloc(m, sizeof(double));
|
||||
// if (mad == NULL) {
|
||||
// perror("calloc failed");
|
||||
// exit(1);
|
||||
// }
|
||||
//
|
||||
// for (long sample = 0; sample < m; sample++) {
|
||||
// resample(values, n, r);
|
||||
//
|
||||
// qsort(r, n, sizeof(double), cmp_double);
|
||||
// double median = r[n / 2];
|
||||
//
|
||||
// for (long i = 0; i < n; i++) {
|
||||
// absdev[i] = fabs(r[i] - median);
|
||||
// }
|
||||
//
|
||||
// qsort(absdev, n, sizeof(double), cmp_double);
|
||||
// mad[sample] = absdev[n / 2];
|
||||
// //printf("mad[%ld] = %e\n", sample, mad[sample]);
|
||||
// }
|
||||
//
|
||||
// double sum = 0.0;
|
||||
// for (long i = 0; i < m; i++)
|
||||
// sum += mad[i];
|
||||
//
|
||||
// double mean = sum / (double) m;
|
||||
// double sumsqr = 0.0;
|
||||
// for (long i = 0; i < m; i++) {
|
||||
// double dev = mad[i] - mean;
|
||||
// sumsqr += dev * dev;
|
||||
// }
|
||||
//
|
||||
// double var = sumsqr / m;
|
||||
// double stdev = sqrt(var);
|
||||
// double sem = stdev / sqrt(m);
|
||||
// double rsem = 100.0 * sem * 1.96 / mean;
|
||||
//
|
||||
// free(mad);
|
||||
// free(absdev);
|
||||
// free(r);
|
||||
//
|
||||
// return rsem;
|
||||
//}
|
||||
|
||||
static void
|
||||
stats(struct sampling *s)
|
||||
{
|
||||
if (s->n < 1)
|
||||
return;
|
||||
|
||||
long outliers = 0;
|
||||
double last = s->samples[s->n - 1];
|
||||
double median = last;
|
||||
double mean = last;
|
||||
double var = NAN;
|
||||
double stdev = NAN;
|
||||
double rstdev = NAN;
|
||||
double sem = NAN;
|
||||
double rsem = NAN;
|
||||
double mad = NAN;
|
||||
//double mad_se = NAN;
|
||||
double q1 = NAN;
|
||||
double q3 = NAN;
|
||||
double iqr = NAN;
|
||||
double pol = NAN;
|
||||
double smin = s->samples[s->n - 1];
|
||||
double smax = s->samples[s->n - 1];
|
||||
|
||||
/* Need at least two samples */
|
||||
if (s->n >= 2) {
|
||||
/* Sort samples to take the median */
|
||||
qsort(s->samples, s->n, sizeof(double), cmp_double);
|
||||
|
||||
double *absdev = calloc(s->n, sizeof(double));
|
||||
if (absdev == NULL) {
|
||||
perror("calloc failed");
|
||||
exit(1);
|
||||
}
|
||||
|
||||
smin = s->samples[0];
|
||||
q1 = s->samples[s->n / 4];
|
||||
median = s->samples[s->n / 2];
|
||||
q3 = s->samples[(s->n * 3) / 4];
|
||||
smax = s->samples[s->n - 1];
|
||||
|
||||
//qcd = (q3 - q1) / (q3 + q1);
|
||||
iqr = q3 - q1;
|
||||
|
||||
double sum = 0.0;
|
||||
for (long i = 0; i < s->n; i++)
|
||||
sum += s->samples[i];
|
||||
|
||||
double n = s->n;
|
||||
mean = sum / n;
|
||||
double sumsqr = 0.0;
|
||||
for (long i = 0; i < s->n; i++) {
|
||||
double x = s->samples[i];
|
||||
double dev = x - mean;
|
||||
sumsqr += dev * dev;
|
||||
absdev[i] = fabs(s->samples[i] - median);
|
||||
//printf("absdev[%3ld] = %e\n", i, absdev[i]);
|
||||
if (x < q1 - 1.5 * iqr || x > q3 + iqr * 1.5)
|
||||
outliers++;
|
||||
}
|
||||
qsort(absdev, s->n, sizeof(double), cmp_double);
|
||||
mad = absdev[s->n / 2] * 1.4826;
|
||||
//mad_se = mad_bootstrap(s->samples, s->n);
|
||||
pol = (double) outliers * 100.0 / n;
|
||||
|
||||
var = sumsqr / n;
|
||||
stdev = sqrt(var);
|
||||
rstdev = 100.0 * stdev / mean;
|
||||
sem = stdev / sqrt(n);
|
||||
rsem = 100.0 * sem * 1.96 / mean;
|
||||
s->rsem = rsem;
|
||||
free(absdev);
|
||||
}
|
||||
|
||||
/* Print the header at the beginning only */
|
||||
if (s->n == 1) {
|
||||
//printf("# --- bench6 ---\n");
|
||||
//printf("# Min %ld runs, max %ld\n", s->nmin, s->nmax);
|
||||
//printf("# Cutoff %%RSEM value set to %f\n", s->min_rsem);
|
||||
//printf("# RUN Number of run\n");
|
||||
//printf("# LAST Value of last run\n");
|
||||
//printf("# MEDIAN Median of values until now\n");
|
||||
//printf("# AVG Mean of values until now\n");
|
||||
//printf("# SD Standard deviation\n");
|
||||
//printf("# %%RSD Relative standard deviation to the mean\n");
|
||||
//printf("# %%RSEM Relative standard error of the mean\n");
|
||||
printf("%4s %5s"
|
||||
" %8s %8s %8s %8s %8s"
|
||||
" %8s %8s %8s"
|
||||
" %5s %5s"
|
||||
" %5s\n",
|
||||
"RUN", "WALL",
|
||||
"MIN", "Q1", "MEDIAN", "Q3", "MAX",
|
||||
"MAD", "IQR", "SD",
|
||||
"%RSD", "%RSEM",
|
||||
"%OUTLIERS");
|
||||
}
|
||||
//RUN WALL LAST MEDIAN AVG SD %RSD %RSEM
|
||||
// 89 125.5 5.085e-03 5.075e-03 5.303e-03 3.500e-03 66.00 7.611
|
||||
//RUN WALL LAST MEDIAN AVG SD %RSD %RSEM
|
||||
// 34 3.0 5.110e-03 5.097e-03 5.121e-03 1.327e-04 2.59 0.87
|
||||
printf(
|
||||
"\r%4ld %5.1f"
|
||||
" %8.2e %8.2e %8.2e %8.2e %8.2e"
|
||||
" %8.2e %8.2e %8.2e"
|
||||
" %5.2f %5.2f"
|
||||
" %5.1f ",
|
||||
s->n, s->wall, /* progress */
|
||||
smin, q1, median, q3, smax, /* centrality */
|
||||
mad, iqr, stdev, /* dispersion */
|
||||
rstdev, rsem, /* rel. dispersion */
|
||||
pol /* outliers */
|
||||
);
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
static int
|
||||
should_continue(struct sampling *s)
|
||||
{
|
||||
stats(s);
|
||||
|
||||
if (s->n < s->nmin)
|
||||
return 1;
|
||||
|
||||
if (s->rsem > s->min_rsem)
|
||||
return 1;
|
||||
|
||||
double dt = bench6_time() - s->t0;
|
||||
if (dt < s->min_time)
|
||||
return 1;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void
|
||||
add_sample(struct sampling *s, double metric, double walltime)
|
||||
{
|
||||
if (s->n >= s->nmax) {
|
||||
die("overflowing samples");
|
||||
} else {
|
||||
s->samples[s->n] = metric;
|
||||
s->n++;
|
||||
s->last = metric;
|
||||
s->wall += walltime;
|
||||
}
|
||||
}
|
||||
|
||||
static int
|
||||
sample(char *argv[])
|
||||
{
|
||||
struct sampling s = { 0 };
|
||||
s.nmax = 4000;
|
||||
s.nmin = 50;
|
||||
s.min_rsem = 0.5;
|
||||
s.min_time = 60.0; /* At least one minute */
|
||||
s.samples = calloc(s.nmax, sizeof(double));
|
||||
s.n = 0;
|
||||
s.name = argv[0];
|
||||
s.t0 = bench6_time();
|
||||
|
||||
while (should_continue(&s)) {
|
||||
double t0 = bench6_time();
|
||||
double metric;
|
||||
if (do_run(argv, &metric) != 0) {
|
||||
err("failed to run benchmark");
|
||||
return 1;
|
||||
}
|
||||
double t1 = bench6_time();
|
||||
double walltime = t1 - t0;
|
||||
|
||||
add_sample(&s, metric, walltime);
|
||||
}
|
||||
|
||||
fprintf(stdout, "\n");
|
||||
|
||||
free(s.samples);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int
|
||||
main(int argc, char *argv[])
|
||||
{
|
||||
progname_set(progname);
|
||||
(void) argc;
|
||||
|
||||
//while (argc && argv && strcmp(argv, "--") != 0) {
|
||||
// argv++;
|
||||
// argc--;
|
||||
//}
|
||||
|
||||
if (sample(argv+1) != 0) {
|
||||
err("failed to sample the benchmark");
|
||||
return 1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
15
test/ci.sh
15
test/ci.sh
@ -1,15 +0,0 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
set -x
|
||||
|
||||
# Allow impure evaluation so we fetch the latest commit from the repo
|
||||
bench6_ref=$(nix build -L --print-out-paths --impure ".#bench6Master")
|
||||
bench6_cur=$(nix build -L --print-out-paths ".#bench6")
|
||||
|
||||
# Add bigotes to the path
|
||||
bigotes=$(nix build --print-out-paths 'jungle#bigotes')
|
||||
export PATH="$bigotes/bin:$PATH"
|
||||
|
||||
bigotes "${bench6_ref}/bin/b6_heat_nanos6" -s 2048 -t 10 -b 64
|
||||
bigotes "${bench6_cur}/bin/b6_heat_nanos6" -s 2048 -t 10 -b 64
|
||||
@ -1,3 +0,0 @@
|
||||
#!/bin/sh
|
||||
|
||||
bigotes b6_heat_nanos6 -s 2048 -t 10 -b 64
|
||||
Loading…
Reference in New Issue
Block a user