#!/bin/sh -x

# File: mp2script
#       Adapted from mp1script by Mark Calleja
#
#   Edit MPDIR and LD_LIBRARY_PATH to suit your local configuration.
# Also don't forget to set the secretword in .mpd.conf.
#

export PWD=`pwd`
export MPD_CONF_FILE=~/.mpd.conf

_CONDOR_PROCNO=$_CONDOR_PROCNO
_CONDOR_NPROCS=$_CONDOR_NPROCS

CONDOR_SSH=`condor_config_val libexec`
CONDOR_SSH=$CONDOR_SSH/condor_ssh

SSHD_SH=`condor_config_val libexec`
SSHD_SH=$SSHD_SH/sshd.sh

. $SSHD_SH $_CONDOR_PROCNO $_CONDOR_NPROCS 

EXECUTABLE=$1
shift

# The binary is copied but the executable flag is cleared,
# so the script has to take care of this.
chmod +x $EXECUTABLE

# Set this to the directory of MPICH2 installation
MPDIR=/usr/local/mpich2
PATH=$MPDIR/bin:.:$PATH
#export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/intel/compilers/lib
export PATH

# Keep track where Condor's scratch dir for this VM is
export SCRATCH_LOC=loclocloc
echo $PWD > ~/$SCRATCH_LOC

finalize()
{
  mpdallexit
  rm ~/$SCRATCH_LOC 
  exit
}
trap finalize TERM

if [ $_CONDOR_PROCNO -ne 0 ]
then
	sleep 5
fi

if [ $_CONDOR_PROCNO -eq 0 ]
then
	CONDOR_CONTACT_FILE=$_CONDOR_SCRATCH_DIR/contact
	export CONDOR_CONTACT_FILE
	
	CONDOR_MPI_SHELL=$(($RANDOM * $RANDOM))
	echo '#!/bin/sh' > $CONDOR_MPI_SHELL 
	echo "cd \`cat ~/$SCRATCH_LOC\`" >> $CONDOR_MPI_SHELL
	echo 'exec $1 $@' >> $CONDOR_MPI_SHELL
	chmod a+r+x $CONDOR_MPI_SHELL

	NODEFILE=nodefile
	myTmp=`cat $CONDOR_CONTACT_FILE | cut -f 2 -d ' ' | sort -u`

        rootHost=`hostname`
	touch $NODEFILE

	for i in $myTmp;
	do
		echo $i >> $NODEFILE
	      	scp $CONDOR_MPI_SHELL $i:/tmp
	done

	nodes=`wc -l $NODEFILE | cut -f 1 -d ' '`

	# The second field in the contact file is the machine name
	# that condor_ssh knows how to use
	sort -n +0 < $CONDOR_CONTACT_FILE | awk '{print $2}' > machines

       	$CONDOR_CHIRP put $_CONDOR_SCRATCH_DIR/contact $_CONDOR_REMOTE_SPOOL_DIR/contact 

	mpdboot -n $nodes -v -f $NODEFILE
	val=$?

	if [ $val -ne 0 ]
	then
      	 	echo "mpdboot error : $val"
       		exit 1
	fi

	mpdtrace -l

	## run the actual mpijob
	mpiexec -machinefile machines -n $_CONDOR_NPROCS /tmp/$CONDOR_MPI_SHELL $EXECUTABLE $@ 
	mpdallexit

	for i in `cat $NODEFILE`;
        do
              ssh $i "rm -f /tmp/$CONDOR_MPI_SHELL"
        done

        rm ~/$SCRATCH_LOC $CONDOR_MPI_SHELL contact nodefile machines
fi


exit $?