#!/bin/sh

# ACE

#################
# PBS VARIABLES #
#################

#PBS -V
#PBS -N abyss
#PBS -e pbs.err
#PBS -o pbs.out
#PBS -r n
##PBS -q <low_mem|mid|mem|high_mem|regular|workq>
#PBS -q high_mem
##PBS -l <free|pack|scatter|vscatter>
#PBS -l place=scatter
##PBS -l select=<1-12>:ncpus=<1-64>:mpiprocs=<1-64>:mem=<gb>:host=<n001-n012>
#PBS -l select=1:ncpus=64:mpiprocs=64:mem=512:host=n010
#PBS -l walltime=8760:00:00
# FABRIC=<shm|sock|ssm|rdma|rdssm>
FABRIC=rdma
CORES=$[ `cat $PBS_NODEFILE | wc -l` ]
NODES=$[ `uniq $PBS_NODEFILE | wc -l` ]
cd $PBS_O_WORKDIR

###################
# LOG AND MODULES #
###################
 
printf "Current time is: `date`\n";
TEND = 
printf "Current PBS work directory is: $PBS_O_WORKDIR\n";
printf "Current PBS queue is: $PBS_O_QUEUE\n"; 
printf "Current PBS job ID is: $PBS_JOBID\n";
printf "Current PBS job name is: $PBS_JOBNAME\n";
printf "PBS stdout log is: $PBS_O_WORKDIR/sgi_mpitest.err\n";
printf "PBS stderr log is: $PBS_O_WORKDIR/sgi_mpitest.log\n";
printf "Fabric interconnect selected is: $FABRIC\n";
printf "This jobs will run on $CORES processors.\n";
. /etc/profile.d/modules.sh
module load mpt && echo "Successfully load modules"
printf "mpiexec_mpt run command location is: `which mpiexec_mpt`\n";
printf "\n[STAT] qstat -f $PBS_JOBID\n";
qstat -f $PBS_JOBID
printf "\n[END] qstat -f $PBS_JOBID\n";
TBEGIN=`echo "print time();" | perl`

###########
# COMMAND #
###########

mpiexec_mpt -n $CORES /apps/poy413mpi -e lepto.poy4.allin.2.0.poy > STD.err 2> STD.out

##########
# FINISH #
##########

TEND=`echo "print time();" | perl`
printf "Job finished: `date`\n";
printf "Job walltime: `expr $TEND - $TBEGIN`\n";