#!@SHELL@
#
# Performs an molecular dynamics simulation with the BOSSANOVA method

#MPIRUN="/opt/packages/mpichgm-1.2.7..15/bin/mpirun.ch_gm"
MPIRUN="/usr/bin/mpirun.mpich"
exec_prefix="@prefix@"
database="@bindir@"
MOLECUILDER="@bindir@/molecuilder"
JOINER="@bindir@/joiner"
CRUNCHER="/mount/bespin/heber/build/mpqc-2.3.0/bin/mpqc"
CONVERTER="/mount/bespin/heber/tmp/mpqc/espack2mpqc.py"
PREPARER="/mount/bespin/heber/tmp/mpqc/convertresults.sh"

function check() 
{
	#1 MESSAGE
	if [ $? -eq 0 ]; then
		if [ -z $1 ]; then
			echo "ok."
		else
			echo "ok: $1."
		fi
	else
		if [ -z $1 ]; then
			echo "failed."
		else
			echo "failed: $1."
		fi
		exit 1
	fi
}

#function MultiRunSim {
#	# 1 is config file dir (with all files)
#	# 2 is the machine file
#
#	${JOBRUNNER} --mpqc ${CRUNCHER} -nprocpernode 2 -nprocperjob 1 -nthreadperproc 2 --threadgrp=posix --messagegrp=proc --memorygrp=proc --nodefile $2 --readdir $1 --inputprefix=${1}/ --outputprefix=${1}/ --autoout --verbose --rerun 2>/dev/stdout	| tee -a dynamic.log
#}

function MultiRunSim {
	# 1 is the number of groups
	# 2 is the directory
	# 3, ... are config files
	
	# find the next free proc group
	divisor=$1
	shift
	DIR=$1
	shift
	started=0
	pwd=`pwd`
	while [ $started -eq 0 ]; do
		groupnr=1
		while [ $groupnr -le $divisor ]; do
			if [ ! -e "${DIR}/ProcRuns${groupnr}" ]; then
				#MaxNodes=`cat ${DIR}/ProcGroup${groupnr} | awk 'END{print NR}'`
				#gamma=`grep ProcPEGamma $1 | awk -F"\t" {'print $2'}`
				#psi=`grep ProcPEPsi $1 | awk -F"\t" {'print $2'}`
				#let nodes=$gamma*$psi
				#if [ $nodes -gt $MaxNodes ]; then
				#	echo "Process $1 needs too many nodes! Breaking." | tee -a dynamic.log
				#	exit 1
				#fi
				nodes=1
				echo "touch ${DIR}/ProcRuns${groupnr}" >"${DIR}/ProcBatch${groupnr}"
				if [ ! -z $1 ]; then
					echo -n "rsh `cat <${DIR}/ProcGroup${groupnr}` 'cd ${pwd}/${DIR}" >>"${DIR}/ProcBatch${groupnr}"
				fi
				while [ ! -z $1 ]; do	# add all config files as single lines
					#echo -n "${MPIRUN} -machinefile ${DIR}/ProcGroup${groupnr} -np $nodes " >>"${DIR}/ProcBatch${groupnr}"
					echo -n "; ${CRUNCHER} -o ${1/conf/out} ${1/conf/in}" >>"${DIR}/ProcBatch${groupnr}"
					shift
				done
				echo "'" >>"${DIR}/ProcBatch${groupnr}"
				echo "rm -f ${DIR}/ProcRuns${groupnr}" >>"${DIR}/ProcBatch${groupnr}"
				/bin/sh "${DIR}/ProcBatch${groupnr}" &
				started=1
				let groupnr=${divisor}+1
			else
				let groupnr=$groupnr+1
			fi
		done
		# wait a few seconds
		#if [ $2 -gt 1 ]; then
		#	sleep 2
		#fi
	done
}

# get command line options
if [ -z $3 ]; then
	echo "Usage: $0 <config file> <Order> <max. bond distance> <MaxNodes> [MaxMDsteps]"
	echo -e "\t<config file> the pcp config file of the total molecule"
	echo -e "\t<Order> the highest bond order (i.e. the cutoff number in ANOVA series expansion)"
	echo -e "\t<max. bond distance> maximum distance to look for bonds (bonds are associated by element covalent radii criterion)"
	echo -e "\t[MaxMDSteps] overrides given MaxOuterStep in config file"
	exit 1;
else
	arg=$1
	mainname=`grep mainname $arg | awk -F"\t" {'print $2'}`
	order=$2
	distance=$3
	if [ -z $4 ]; then
		MaxSteps=`grep MaxOuterStep $arg | awk -F"\t" {'print $2'}`
	else
		MaxSteps=$4
	fi
	echo "Going to run for a total of $MaxSteps steps, bond order $order and maximum distance $distance of config file $arg." | tee -a dynamic.log
fi


# get the directory
DIR=`dirname $arg`
if [ -z "`grep $DIR $arg`" ]; then
	echo "Cannot find the directory $DIR in the config file." | tee -a dynamic.log
	exit 1;
else
	echo "Using $DIR as directory." | tee -a dynamic.log
fi

PBS_NODEFILE="${DIR}/machines"
if [ ! -e $PBS_NODEFILE ]; then
	i=1
	cpus=`cat /proc/cpuinfo | grep processor | wc -l`
	echo "localhost" >$PBS_NODEFILE
	while [ $i -lt $cpus ]; do      # add one localhost per cpu to machines file
		echo "localhost" >>$PBS_NODEFILE
		let i=$i+1
	done
fi

# delete old processor group files
rm ${DIR}/ProcGroup* -f
rm ${DIR}/ProcRuns* -f
rm ${DIR}/ProcBatch* -f

# put nodes into groups
MaxNodes=0
for node in `cat <$PBS_NODEFILE`; do
	let MaxNodes=$MaxNodes+1
done
gamma=`grep ProcPEGamma $arg | awk -F"\t" {'print $2'}`
psi=`grep ProcPEPsi $arg | awk -F"\t" {'print $2'}`
let nodes=$gamma*$psi
let divisor=$MaxNodes/$nodes
echo "Using $divisor processor groups." | tee -a dynamic.log
nodenr=0
groupnr=1
for node in `cat <$PBS_NODEFILE`; do
	let nodenr=$nodenr+1
	#echo "Current node $nodenr is $node." | tee -a dynamic.log
	let currentgrouplimit=$groupnr*$nodes
	if [ $currentgrouplimit -lt $nodenr ]; then
		let groupnr=$groupnr+1
	fi
	#echo "Putting into group $groupnr." | tee -a dynamic.log
	echo "$node" >>"${DIR}/ProcGroup${groupnr}"
done
i=0
while [ $i -lt $groupnr ]; do
	let i=$i+1
	echo "Group nr. $i" | tee -a dynamic.log
	echo "===========" | tee -a dynamic.log
	cat <"${DIR}/ProcGroup${i}"
	cat <"${DIR}/ProcGroup${i}" >>dynamic.log
	echo -e "\n" | tee -a dynamic.log
done

# copy first conf
cp $arg ${arg}.MD


i=1;
while [ $i -le $MaxSteps ]; do
# break down the molecule with molecuilder
	echo -n "Fragmenting ... " | tee -a dynamic.log
  ${MOLECUILDER} ${arg}.MD -e ${database} -f $distance $order 2>/dev/null >/dev/null
	check | tee -a dynamic.log
	echo "done." | tee -a dynamic.log

# get the number of digits of the fragment count
	digits=1
	while [ ! -e ${DIR}/BondFragment`printf "%0${digits}d" 0`.conf ]; do
		let digits=$digits+1
	done
	echo "Found $digits digits for the fragment number." | tee -a dynamic.log

# get the fragment count
	frag=0
	while [ -e ${DIR}/BondFragment`printf "%0${digits}d" $frag`.conf ]; do
		# unset MaxOuterStep in config file
		sed -i -e "s#MaxOuterStep.*\##MaxOuterStep\t0\t\##" ${DIR}/BondFragment`printf "%0${digits}d" $frag`.conf
		rm -rf ${DIR}/BondFragment`printf "%0${digits}d" $frag`
		let frag=$frag+1
	done
	echo "There are $frag fragments." | tee -a dynamic.log


# evaluate each fragment
#  j=0
#  while [ $j -lt $frag ]; do
#		number=`printf "%0${digits}d" $j`
#		# convert all configs
#		echo -n "Converting ${DIR}/BondFragment${number}.conf ..." | tee -a dynamic.log
#		sh $CONVERTER ${DIR}/BondFragment${number}.conf
#		check | tee -a dynamic.log
#    let j=$j+1
#	done
#
#	MultiRunSim ${DIR} $PBS_NODEFILE
#
#  j=0
#	while [ $j -lt $frag ]; do
#		number=`printf "%0${digits}d" $j`
#		# rename output files
#		echo -n "Renaming `ls ${DIR}/BondFragment${number}.out.001.02.02` ..." | tee -a dynamic.log
#		mv ${DIR}/BondFragment${number}.out.001.02.02 ${DIR}/BondFragment${number}.out
#		check | tee -a dynamic.log
#    let j=$j+1
#	done

	# reset command arrays
  grp=0;
  while [ $grp -lt $divisor ]; do
		command[$grp]=""
    let grp=$grp+1
  done

	# distribute the jobs among the groups
  j=0;
  while [ $j -lt $frag ]; do
		number=`printf "%0${digits}d" $j`
		# convert all configs
		#echo -n "Converting ${DIR}/BondFragment${number}.conf ..." | tee -a dynamic.log
		#sh $CONVERTER ${DIR}/BondFragment${number}.conf
		#check | tee -a dynamic.log
		# and distribute
		let grp=${j}%${divisor}
		#echo "BondFragment${number}.conf is evaluated by group $grp."
		command[$grp]="${command[$grp]}BondFragment${number}.conf "
    let j=$j+1
  done

  # go through all groups and run the job
  olddivisor=$divisor
  if [ $divisor -gt $frag ]; then
	divisor=$frag
  fi
  grp=0;
  while [ $grp -lt $divisor ]; do
		number=`printf "%0${digits}d" $j`
		echo -n  "Starting calculation of group $grp with fragments \"${command[$grp]}\" at step $i ... " | tee -a dynamic.log
		MultiRunSim $divisor ${DIR} ${command[$grp]}
		echo "done." | tee -a dynamic.log
    let grp=$grp+1
	done
	divisor=$olddivisor	

# wait till all ProcRuns files are gone
#	if [ $divisor -gt 1 ]; then
		echo "Waiting for all running jobs at step $i to end ... " | tee -a dynamic.log
		while [ ! -z "`find ${DIR} -name 'ProcRuns*'`" ]; do
			#if [ ! -z "`find ${DIR} -name 'ProcRuns*'`" ]; then
			#	echo "still `ls ${DIR}/ProcRuns*` present"
			#fi
			sleep 1
		done
		echo "done." | tee -a dynamic.log
#	fi


# convert results
	sleep 1		# necessary for result files to close
	echo -n "Converting all results ... " | tee -a dynamic.log
	sh $PREPARER $DIR
	check | tee -a dynamic.log

# join the resulting forces into a single file
	echo -n "Joining fragment energies ... " | tee -a dynamic.log
  ${JOINER} ${DIR}/ $mainname >/dev/null 2>/dev/null
	check | tee -a dynamic.log
	echo "done." | tee -a dynamic.log

# move the ions by calling pcp with this force file
	sed -i -e "s#MaxOuterStep.*\##MaxOuterStep\t$i\t\##" ${arg}.MD
	echo -n "Moving ions with obtained forces at step $i ... " | tee -a dynamic.log
	$MOLECUILDER ${arg}.MD -e ${database} -P "${DIR}/pcp.Order${order}.forces.all"  2>/dev/null >/dev/null
	echo "done" | tee -a dynamic.log

# last of all, put "joined" energy and forces under this step
	cp ${DIR}/pcp.Order${order}.energy.all ${DIR}/pcp.step${i}.energy.all
	cp ${DIR}/pcp.Order${order}.forces.all ${DIR}/pcp.step${i}.forces.all
  
# next step
	let i=$i+1
done

exit 0