#!/bin/sh
##**************************************************************
##
## Copyright (C) 1990-2007, Condor Team, Computer Sciences Department,
## University of Wisconsin-Madison, WI.
## 
## Licensed under the Apache License, Version 2.0 (the "License"); you
## may not use this file except in compliance with the License.  You may
## obtain a copy of the License at
## 
##    http://www.apache.org/licenses/LICENSE-2.0
## 
## Unless required by applicable law or agreed to in writing, software
## distributed under the License is distributed on an "AS IS" BASIS,
## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
## See the License for the specific language governing permissions and
## limitations under the License.
##
##**************************************************************


_CONDOR_PROCNO=$_CONDOR_PROCNO
echo $_CONDOR_PROCNO
_CONDOR_NPROCS=$_CONDOR_NPROCS
echo $_CONDOR_NPROCS

MPDIR=/gpfs/apps/Intel/impi/3.1
PATH=$MPDIR/bin:.:$PATH
#export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/gpfs/apps/Intel/impi/3.1/lib
#export PATH


CONDOR_SSH=`condor_config_val libexec`
CONDOR_SSH=$CONDOR_SSH/condor_ssh

SSHD_SH=`condor_config_val libexec`
SSHD_SH=$SSHD_SH/sshd.sh

. $SSHD_SH $_CONDOR_PROCNO $_CONDOR_NPROCS 

# If not the head node, just sleep forever, to let the
# sshds run
if [ $_CONDOR_PROCNO -ne 0 ]
then
		wait
		sshd_cleanup
		exit 0
fi
echo "machine 0"
echo `hostname`
EXECUTABLE=$1
echo $EXECUTABLE
shift

# the binary is copied but the executable flag is cleared.
# so the script have to take care of this
#chmod +x $EXECUTABLE

# Set this to the bin directory of MPICH installation
#MPDIR=/u/g/t/gthain/mpich-1.2.6/bin
#PATH=$MPDIR:.:$PATH
#export PATH

export P4_RSHCOMMAND=$CONDOR_SSH

CONDOR_CONTACT_FILE=$_CONDOR_SCRATCH_DIR/contact
export CONDOR_CONTACT_FILE
cat $CONDOR_CONTACT_FILE

# The second field in the contact file is the machine name
# that condor_ssh knows how to use
cat $CONDOR_CONTACT_FILE | cut -f 2 -d ' ' | sort -u > hostfile
cat $CONDOR_CONTACT_FILE | sort | cut -f 2 -d ' '   > machines



cat hostfile
cat machines
echo `which mpirun`
echo `hostname`
echo $PATH
## run the actual mpijob
#echo "mpirun -f hostfile -np $_CONDOR_NPROCS -machinefile machines $EXECUTABLE $@ "
#echo "mpirun -f hostfile -machinefile machines -env I_MPI_DEBUG 10 -np $_CONDOR_NPROCS  $EXECUTABLE $@"
echo "mpirun -f hostfile -machinefile machines -genv I_MPI_DEVICE sock -genv I_MPI_DEBUG 10 -np $_CONDOR_NPROCS  $EXECUTABLE $@"
mpirun -f hostfile -machinefile machines -genv I_MPI_DEVICE sock -genv I_MPI_DEBUG 10 -np $_CONDOR_NPROCS  $EXECUTABLE $@ > /install/home/zhaokun/result
echo "mpiexec over"
sshd_cleanup
#rm -f machines

exit $?