#!/bin/bash

source comp_intel_env

OPAL_PREFIX=/afs/ics.muni.cz/home/wiesner/openmpi-1.6.3-intel-2011-x86_64
mpirun_bin=$OPAL_PREFIX/bin/mpirun
sander_bin=/afs/ics.muni.cz/home/wiesner/sander_modif.MPI

tar xfj ${OPAL_PREFIX%/*}/ompi-1.6.3.tar.bz2
cp -u $sander_bin .

OPAL_PREFIX=$PWD/${OPAL_PREFIX##*/}
LD_LIBRARY_PATH="$OPAL_PREFIX/lib:$LD_LIBRARY_PATH"
PATH="$OPAL_PREFIX/bin:$PATH"
mpirun_bin=$OPAL_PREFIX/bin/mpirun
sander_bin=${sander_bin##*/}

echo
echo $LD_LIBRARY_PATH
echo
echo


hname=`hostname`
allh="`grep -v "$hname" $PBS_NODEFILE | sort -u`"

# copy all data to slave nodes
for machine in $allh; do
  ssh $machine "mkdir -vp $PWD"
  scp -r * ${machine}:$PWD
done
# copy mdin as the last one
sleep 2
for machine in $allh; do
  scp -r mdin ${machine}:$PWD
done



$mpirun_bin  -x OPAL_PREFIX -x LD_LIBRARY_PATH -x PATH  \
             -np $INF_NCPU  --hostfile  $PBS_NODEFILE  \
             --wdir $PWD   \
   \
   $sander_bin -ng 16 -groupfile group_file



# copy all data from the slave nodes back
for machine in $allh; do
  echo File listing of $machine :
  ssh $machine "cd $PWD; ls -lh"
  echo

  # get only those files which are older than mdin
  files="`ssh $machine "cd $PWD; find -maxdepth 1 -newer mdin" | cut -d/ -f2 | \
            grep -v "^run_refep\|^\.\$" | sort -u`"
  echo Getting these files from $machine :
  echo $files
  echo
  for f in $files; do
    scp -r ${machine}:$PWD/$f .
  done

  ssh ${machine} "rm -rf $PWD"
done

for machine in . $allh; do
  rm -rf $machine/${OPAL_PREFIX##*/} $machine/$sander_bin
done