55 lines
1.1 KiB
Bash
Executable File
55 lines
1.1 KiB
Bash
Executable File
#!/usr/bin/bash
|
|
|
|
# There are several situations in which we may find the jobs:
|
|
# - There are no jobs queued or running
|
|
# - There is at least one job running
|
|
# - There is one job queued
|
|
# - There was a job running but ended and is now ending
|
|
|
|
set -x
|
|
set -e
|
|
|
|
path="$1"
|
|
allocated=
|
|
|
|
# First determine if we already have jobs already
|
|
n=$(squeue --me -lh | wc -l)
|
|
|
|
if [ "$n" -gt 1 ]; then
|
|
echo "Too many jobs queued already" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [ "$n" == 0 ]; then
|
|
# No running jobs, so allocate a new job
|
|
salloc -N 1 --constraint=dmaqdma --no-shell -t 1-00
|
|
allocated=1
|
|
|
|
# Wait until the job is running
|
|
while [ "$n" != 1 ]; do
|
|
sleep 2
|
|
n=$(squeue --me -lh | grep RUNNING | wc -l)
|
|
done
|
|
else
|
|
# There is one job, ensure it is running
|
|
n=$(squeue --me -lh | grep RUNNING | wc -l)
|
|
|
|
if [ "$n" != 1 ]; then
|
|
echo "The job is not running, stopping" >&2
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
# If this point is reached there is one job running
|
|
|
|
host=$(squeue -h -o %N)
|
|
echo "Switching to $host"
|
|
|
|
# Continue the execution there
|
|
ssh "$host" "$path/run-node.sh" "$path"
|
|
|
|
# Cancel our job if it was successful
|
|
if [ "$allocated" ]; then
|
|
scancel --me
|
|
fi
|