#!/bin/csh -f
#PBS -l nodes=4

# this is a sample script to run an mpi job with large data
# the data will be broadcast from /data filesystem on master to local
# /scratch filesystems on the nodes

# Janine Kennedy
# September 30, 2002

# set work directory,  relative to home
# and executable and data file
set workdir=$HOME/temp

# display what we got from PBS
printenv

# list of nodes allocated by PBS to the job
set nodes=(`cat $PBS_NODEFILE`)
echo job is running on $nodes

#build nodelist with 2 cpus per node
set nodelist
foreach node ($nodes) 
    set nodelist = ($nodelist $node 2) 
end 
echo the node list is $nodelist

#your file, will be accessed on master
set in_data=$workdir/big.data

# the local copy, will be accessed on computational nodes, MUST be in /scratch
set out_data=/scratch/$USER/big.data
echo in_data $in_data out_data $out_data

# the copying runs from master; this maybe changed to call fbcast_mpich
mpimon /usr/local/bin/fbcast_scali $in_data $out_data -- master $nodes
echo copying data done

# your executable file a.out, will be accessed on master
set in_exe=$workdir/a.out

# the local copy accessed on computational nodes, MUST be in /scratch
set out_exe=/scratch/$USER/a.out

# get executable command
mpimon /usr/local/bin/fbcast_scali $in_exe $out_exe -- master $nodes
echo copying executable done

#run executable
mpimon $out_exe  -- $nodelist 

#clean up files from scratch
foreach node ($nodes)
        rsh $node "rm $out_exe $out_data"
end