#!/bin/bash -l

#PBS -S /bin/bash
#PBS -l walltime=08:00:00
#PBS -l procs=60
#PBS -o mppnp.out
#PBS -e mppnp.err
#PBS -l mem=30gb
#PBS -l epilogue=epilogue.script
cd $PBS_O_WORKDIR

export OMP_NUM_THREADS=1
export MKL_NUM_THREADS=1

JOBINFO=mppnp.${PBS_JOBID}
echo "Starting run at: `date`" > $JOBINFO
echo "Current working directory is `pwd`" >> $JOBINFO
echo $PBS_NODEFILE >>$JOBINFO



echo "Node file: $PBS_NODEFILE :" >>$JOBINFO
echo "---------------------" >>$JOBINFO
cat $PBS_NODEFILE >>$JOBINFO
echo "---------------------" >>$JOBINFO

MPIEX=mpiexec
#in later versions where mppnp.exe is in run dir
#FULL_EXE=$PBS_O_WORKDIR/mppnp.exe

FULL_EXE=ppn_dir

# create output directories if needed
[ -d H5_surf ]    || mkdir H5_surf
[ -d H5_out ]     || mkdir H5_out
[ -d H5_restart ] || mkdir H5_restart


NUM_PROCS=`/bin/awk 'END {print NR}' $PBS_NODEFILE` >>$JOBINFO
echo "Running on $NUM_PROCS processors." >>$JOBINFO

echo "Do the mounting at: `date`" >>$JOBINFO

vosFolder=./vos_cadc_`echo $PBS_JOBID|awk -F. '{print $1}'`
vos_log=./vos_log_`echo $PBS_JOBID|awk -F. '{print $1}'`
vos_cache=./vos_cache_`echo $PBS_JOBID|awk -F. '{print $1}'`
mountvofs --debug --cache_nodes --log=$vos_log --cache_dir=$vos_cache --mountpoint=$vosFolder --vospace=vos:nugrid >> $PBS_O_WORKDIR/$JOBINFO
sleep 5
echo "VOS folder mounted $vosFolder" >> $PBS_O_WORKDIR/$JOBINFO
ls ./vos_cadc_`echo $PBS_JOBID|awk -F. '{print $1}'` >> $PBS_O_WORKDIR/$JOBINFO

echo "Add mounting point to ppn_frame.input" >>$PBS_O_WORKDIR/$JOBINFO
cp ppn_frame.input_template ppn_frame.input
sed -i 's,''cadc,'"$vosFolder"',g' ppn_frame.input
ls `grep datdir ppn_frame.input|awk -F= '{print $2}'`  >>$PBS_O_WORKDIR/$JOBINFO

echo "Starting run at: `date`" >>$PBS_O_WORKDIR/$JOBINFO

$MPIEX -n ${NUM_PROCS} $FULL_EXE >>$PBS_O_WORKDIR/$JOBINFO

##$FULL_EXE -n ${NUM_PROCS} $FULL_EXE >>$PBS_O_WORKDIR/$JOBINFO

echo "Program mppnp finished with exit code $? at: `date`" >>$PBS_O_WORKDIR/$JOBINFO