On gaia, it was found easiest to use two scripts. The first, the 'submission' script, requests resources from the cluster and runs the second 'launch' script on the nodes assigned:
#!/bin/bash ##number of nodes nodes=2 oarsub --resource "nodes=$nodes,walltime=72"\ "./launch.bash $(pwd)/START_CONFIGS/start_2162525824.blk $(pwd)/bench.conf"
#!/bin/bash ##this should point to directory which contains the espresso job script HARNESS=/scratch/users/jberryman/lj_spres/02197_particles_lange_0.001/harness ##this should point to a the FRESHS script directory root FRESHS=/home/clusterusers/jberryman/freshs ##Simulation executable ESPRESSO=/home/clusterusers/jberryman/Espresso/bin/Espresso ##read/write DBs here DB=/home/clusterusers/jberryman/lj_spres/02197_particles_lange_0.001/DB ##read/write system states here CONF=/scratch/users/jberryman/lj_spres/02197_particles_lange_0.001/CONF ##list of node hostnames NODES=$(cat $OAR_NODEFILE) ##save pwd HERE=$(pwd) ##test arguments if [ $# -lt 2 ] then echo 'Error: require 2 arguments, name of an initial system state file' echo ' +FRESHS config file.' exit 65 fi startFile="$1" ##this should point to a FRESHS config file inFile="$2" ##test if Espresso is present if [ $(which "$ESPRESSO") ] then echo have Espresso as: "$ESPRESSO" else echo could not find Espresso as: "$ESPRESSO" exit fi if [ $(ls "$FRESHS"/server/server.py) ] then echo have FRESHS: "$FRESHS" else echo could not find FRESHS as: "$FRESHS" exit fi ##prepare start config cp $startFile $HARNESS/initial_config.dat ##uncomment to turn on profiling #PROF=" -m cProfile -o profile.dat" PROF="" ##clip the first and last host ids from the list: SERVER_HOST=$(echo $NODES | awk '{print $1}') LAST_NODE=$(echo $NODES | awk '{print $NF}') NODES=$(echo $NODES | awk '{for(i=2;i<NF;i++){printf "%s ",$i}}') ##launch the server oarsh $SERVER_HOST "cd $HERE; python $PROF $FRESHS/server/main_server.py \ --db-folder $DB --config-folder $CONF --config $inFile > server.log 2>&1" & sleep 2 ##wait for the server to load before starting the clients for node_host in $NODES do oarsh "$node_host" "cd $HERE; nice python $FRESHS/client/main_client.py --server $SERVER_HOST \ $ESPRESSO $HARNESS >/dev/null 2>/dev/null" & done oarsh "$LAST_NODE" "cd $HERE; nice python $FRESHS/client/main_client.py --server $SERVER_HOST \ $ESPRESSO $HARNESS >client_log 2>&1"