From 1d015c7e1edb6ed4fe6cc403675a753516bc0d6b Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 1 Mar 2021 12:00:58 +0100 Subject: [PATCH] ds: add osu fast generators --- garlic/ds/index.nix | 5 +++ garlic/ds/osu/bw.nix | 23 ++++++++++++++ garlic/ds/osu/bw.py | 64 +++++++++++++++++++++++++++++++++++++++ garlic/ds/osu/latency.nix | 23 ++++++++++++++ garlic/ds/osu/latency.py | 64 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 179 insertions(+) create mode 100644 garlic/ds/osu/bw.nix create mode 100644 garlic/ds/osu/bw.py create mode 100644 garlic/ds/osu/latency.nix create mode 100644 garlic/ds/osu/latency.py diff --git a/garlic/ds/index.nix b/garlic/ds/index.nix index 89d2907..c3c0454 100644 --- a/garlic/ds/index.nix +++ b/garlic/ds/index.nix @@ -10,4 +10,9 @@ std = { timetable = callPackage ./std/timetable.nix {}; }; + + osu = { + latency = callPackage ./osu/latency.nix {}; + bw = callPackage ./osu/bw.nix {}; + }; } diff --git a/garlic/ds/osu/bw.nix b/garlic/ds/osu/bw.nix new file mode 100644 index 0000000..9426998 --- /dev/null +++ b/garlic/ds/osu/bw.nix @@ -0,0 +1,23 @@ +{ + stdenv +, python3 +, gzip +}: + +resultTree: + +stdenv.mkDerivation { + name = "osu-bw.json.gz"; + preferLocalBuild = true; + src = ./bw.py; + phases = [ "unpackPhase" "installPhase" ]; + + unpackPhase = '' + cp $src bw.py + ''; + + buildInputs = [ python3 gzip ]; + installPhase = '' + python bw.py ${resultTree} | gzip > $out + ''; +} diff --git a/garlic/ds/osu/bw.py b/garlic/ds/osu/bw.py new file mode 100644 index 0000000..4b1df43 --- /dev/null +++ b/garlic/ds/osu/bw.py @@ -0,0 +1,64 @@ +import json, re, sys, os, glob +from os import path + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, flush=True, **kwargs) + +def process_run(tree, runPath): + with open("stdout.log", "r") as f: + lines = [line.strip() for line in f.readlines()] + + for line in lines: + + if not re.match('^[0-9]+ *[0-9\.]+$', line): + continue + + slices = line.split() + size = slices[0] + bw = slices[1] + + tree['size'] = int(size) + tree['bw'] = float(bw) + print(json.dumps(tree)) + +def process_result_tree(resultTree): + + eprint("processing resultTree: " + resultTree) + + os.chdir(resultTree) + + experiments = glob.glob(resultTree + "/*-experiment") + + for exp in glob.glob("*-experiment"): + eprint("found experiment: " + exp) + expPath = path.join(resultTree, exp) + os.chdir(expPath) + + for unit in glob.glob("*-unit"): + eprint("found unit: " + unit) + unitPath = path.join(resultTree, exp, unit) + os.chdir(unitPath) + + with open('garlic_config.json') as json_file: + garlic_conf = json.load(json_file) + + tree = {"exp":exp, "unit":unit, "config":garlic_conf} + + for i in range(garlic_conf['loops']): + run = str(i + 1) + runPath = path.join(resultTree, exp, unit, run) + if path.isdir(runPath) == False: + eprint("missing run {}, aborting".format(run)) + exit(1) + + tree["run"] = run + os.chdir(runPath) + + process_run(tree, runPath) + + +if len(sys.argv) != 2: + eprint("usage: python {} ".format(argv[0])) + exit(1) + +process_result_tree(sys.argv[1]) diff --git a/garlic/ds/osu/latency.nix b/garlic/ds/osu/latency.nix new file mode 100644 index 0000000..3fe0426 --- /dev/null +++ b/garlic/ds/osu/latency.nix @@ -0,0 +1,23 @@ +{ + stdenv +, python3 +, gzip +}: + +resultTree: + +stdenv.mkDerivation { + name = "osu-latency.json.gz"; + preferLocalBuild = true; + src = ./latency.py; + phases = [ "unpackPhase" "installPhase" ]; + + unpackPhase = '' + cp $src latency.py + ''; + + buildInputs = [ python3 gzip ]; + installPhase = '' + python latency.py ${resultTree} | gzip > $out + ''; +} diff --git a/garlic/ds/osu/latency.py b/garlic/ds/osu/latency.py new file mode 100644 index 0000000..2df08ee --- /dev/null +++ b/garlic/ds/osu/latency.py @@ -0,0 +1,64 @@ +import json, re, sys, os, glob +from os import path + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, flush=True, **kwargs) + +def process_run(tree, runPath): + with open("stdout.log", "r") as f: + lines = [line.strip() for line in f.readlines()] + + for line in lines: + + if not re.match('^[0-9]+ *[0-9\.]+$', line): + continue + + slices = line.split() + size = slices[0] + latency = slices[1] + + tree['size'] = int(size) + tree['latency'] = float(latency) + print(json.dumps(tree)) + +def process_result_tree(resultTree): + + eprint("processing resultTree: " + resultTree) + + os.chdir(resultTree) + + experiments = glob.glob(resultTree + "/*-experiment") + + for exp in glob.glob("*-experiment"): + eprint("found experiment: " + exp) + expPath = path.join(resultTree, exp) + os.chdir(expPath) + + for unit in glob.glob("*-unit"): + eprint("found unit: " + unit) + unitPath = path.join(resultTree, exp, unit) + os.chdir(unitPath) + + with open('garlic_config.json') as json_file: + garlic_conf = json.load(json_file) + + tree = {"exp":exp, "unit":unit, "config":garlic_conf} + + for i in range(garlic_conf['loops']): + run = str(i + 1) + runPath = path.join(resultTree, exp, unit, run) + if path.isdir(runPath) == False: + eprint("missing run {}, aborting".format(run)) + exit(1) + + tree["run"] = run + os.chdir(runPath) + + process_run(tree, runPath) + + +if len(sys.argv) != 2: + eprint("usage: python {} ".format(argv[0])) + exit(1) + +process_result_tree(sys.argv[1])