#!/bin/bash

export mpiDebug=1

[ ${mpiDebug} == 1 ] && export verbose='-v'

export CONDOR_CHIRP=`condor_config_val libexec`/condor_chirp
export P4_RSHCOMMAND=`condor_config_val libexec`/mpi_ssh


EXECUTABLE=$1
WORKING_DIR=`dirname ${EXECUTABLE}`
#EXECUTABLE=./`basename ${EXECUTABLE}`
shift


## Write info into common spool directory
echo "${_CONDOR_PROCNO} `hostname` ${_CONDOR_SCRATCH_DIR} ${WORKING_DIR} ${EXECUTABLE}" | ${CONDOR_CHIRP} put -mode cwa - ${_CONDOR_REMOTE_SPOOL_DIR}/contact
#echo "${_CONDOR_PROCNO} `hostname` ${_CONDOR_SCRATCH_DIR} ${WORKING_DIR} ${EXECUTABLE}" > ./aa
#${CONDOR_CHIRP} put ./aa ${_CONDOR_REMOTE_SPOOL_DIR}/contact
#/bin/rm ./aa

[ ${mpiDebug} == 1 ] && echo Running ${_CONDOR_PROCNO} / ${_CONDOR_NPROCS} at ${_CONDOR_SCRATCH_DIR} as `id -un` in `hostname`

## I'm server, get info from common spool directory
mpiLock=${WORKING_DIR}/mpi_lock.`basename ${_CONDOR_REMOTE_SPOOL_DIR}`
touch ${mpiLock}

if [ ${_CONDOR_PROCNO} -eq 0 ]; then
	while true;
	do
		${CONDOR_CHIRP} fetch ${_CONDOR_REMOTE_SPOOL_DIR}/contact ${_CONDOR_SCRATCH_DIR}/contact
		lines=`wc -l ${_CONDOR_SCRATCH_DIR}/contact | awk '{print $1}'`
		if [ $lines -eq ${_CONDOR_NPROCS} ]; then
			break
		else
			sleep 1
		fi
	done
	${CONDOR_CHIRP} remove ${_CONDOR_REMOTE_SPOOL_DIR}/contact

	sort -n +0 < ${_CONDOR_SCRATCH_DIR}/contact | awk '{print $2}' > ${_CONDOR_SCRATCH_DIR}/machines

	[ ${mpiDebug} == 1 ] && echo --- contact
	[ ${mpiDebug} == 1 ] && cat contact
	[ ${mpiDebug} == 1 ] && echo --- machines
	[ ${mpiDebug} == 1 ] && cat machines
else
#    sleep 3
#    wait

    while [ -e ${mpiLock} ];
    do
    	sleep 3
    done
#    wait

	exit 0
fi



###############
### Initial ###
###############
export PATH=/usr/local/mpich-1.2.7p1/bin
export PATH=${PATH}:/bin:/sbin
export PATH=${PATH}:/usr/bin:/usr/sbin
export PATH=${PATH}:/usr/local/bin:/usr/local/sbin

export LD_LIBRARY_PATH=/usr/local/mpich-1.2.7p1/lib
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/lib:/usr/lib:/usr/local/lib


[ ${mpiDebug} == 1 ] && echo --- which mpirun
[ ${mpiDebug} == 1 ] && which mpirun

[ ${mpiDebug} == 1 ] && echo --- printenv
[ ${mpiDebug} == 1 ] && printenv

cd ${WORKING_DIR}

[ ${mpiDebug} == 1 ] && echo --- test
[ ${mpiDebug} == 1 ] && echo mpirun ${verbose} -t -np ${_CONDOR_NPROCS} -machinefile ${_CONDOR_SCRATCH_DIR}/machines ${EXECUTABLE} $@
[ ${mpiDebug} == 1 ] &&      mpirun ${verbose} -t -np ${_CONDOR_NPROCS} -machinefile ${_CONDOR_SCRATCH_DIR}/machines ${EXECUTABLE} $@

[ ${mpiDebug} == 1 ] && echo --- run
[ ${mpiDebug} == 1 ] && echo mpirun ${verbose} -np ${_CONDOR_NPROCS} -machinefile ${_CONDOR_SCRATCH_DIR}/machines ${EXECUTABLE} $@
                             mpirun ${verbose} -np ${_CONDOR_NPROCS} -machinefile ${_CONDOR_SCRATCH_DIR}/machines ${EXECUTABLE} $@
export s=$?


/bin/rm -f ${_CONDOR_SCRATCH_DIR}/contact
/bin/rm -f ${_CONDOR_SCRATCH_DIR}/machines
/bin/rm -f ${mpiLock}

exit $s