#!/usr/bin/bash # There are several situations in which we may find the jobs: # - There are no jobs queued or running # - There is at least one job running # - There is one job queued # - There was a job running but ended and is now ending set -x set -e path="$1" allocated= # First determine if we already have jobs already n=$(squeue --me -lh | wc -l) if [ "$n" -gt 1 ]; then echo "Too many jobs queued already" >&2 exit 1 fi if [ "$n" == 0 ]; then # No running jobs, so allocate a new job salloc -N 1 --constraint=dmaqdma --no-shell -t 1-00 allocated=1 # Wait until the job is running while [ "$n" != 1 ]; do sleep 2 n=$(squeue --me -lh | grep RUNNING | wc -l) done else # There is one job, ensure it is running n=$(squeue --me -lh | grep RUNNING | wc -l) if [ "$n" != 1 ]; then echo "The job is not running, stopping" >&2 exit 1 fi fi # If this point is reached there is one job running host=$(squeue -h -o %N) echo "Switching to $host" # Continue the execution there ssh "$host" "$path/run-node.sh" "$path" # Cancel our job if it was successful if [ "$allocated" ]; then scancel --me fi