#!/bin/bash
login=`whoami`
date=`date +%Y%m%d_%H%M%S`
rev=$(git log|head -n 100|grep git-svn-id|head -n 1|awk -F@ '{print $2}'|awk '{print $1}')

CMIN=20
DEFAULT_ORDERING="API_NO"
ROOT=${HOME}/ricar/src
USER=`whoami`
OUTPUTDIR=/lustre/$USER/
machine=mirage
LATE="API_NO"
FRONTAL_NAME=`uname -n | sed -e 's/[0-9]//g'`
if [ "$FRONTAL_NAME" = "curie" ]
then
    MPIRUN=ccc_mprun
else
    MPIRUN="mpirun -pernode -np \${nmpi} "
fi

paths=(
    ~/matrices/Serena.mtx
    ~/matrices/Flan1565.mtx
    ~/matrices/Geo1438.mtx
    ~/matrices/Hook.mtx
    ~/matrices/audi.rsa
    ~/matrices/afshell10.mtx
    ~/matrices/FilterV2.mtx
    ~/matrices/G3.mtx
    ~/matrices/matr5.rua
    ~/matrices/matrix_acoustic_pml_DF.mtx
    ~/matrices/10M_Matrice
)

drivers=(
    mm
    mm
    mm
    mm
    rsa
    mm
    mm
    mm
    rsa
    mm
    mm
)

facto=(
    LDLT
    LLT
    LLT
    LU
    LLT
    LU
    LU
    LLT
    LU
    LDLT
    LDLT
)

binaries=(
    dsimple
    dsimple
    dsimple
    dsimple
    dsimple
    dsimple
    zsimple
    dsimple
    dsimple
    zsimple
    zsimple
)

binaries_cmake=(
    d_simple
    d_simple
    d_simple
    d_simple
    d_simple
    d_simple
    z_simple
    d_simple
    d_simple
    z_simple
    z_simple
)

matrices=(
    Serena    # 0
    Flan1565  # 1
    Geo1438   # 2
    Hook      # 3
    audi      # 4
    afshell10 # 5
    FilterV2  # 6
    G3        # 7
    MHD       # 8
    pml_DF    # 9
    10Million # 10
)


scheds=( pastix  eager   dmda    ws       ws      lws      lws     lws      lws    )
maxcuda=(0       3       3       3        3       3        3       3        3      )
starpu=( API_NO  API_YES API_YES API_YES  API_YES API_YES  API_YES API_YES  API_YES)
fanin=(  0       1       1       1        0       1        0       1        0      )
nested=( 0       0       0       0        0       0        0       1        1      )
#scheds=( pastix  eager   dmda    lws      lws      prio    prio    )
#maxcuda=(0       3       3       3        3        3       3       )
#starpu=( API_NO  API_YES API_YES API_YES  API_YES  API_YES API_YES )
#fanin=(  0       1       1       1        0        1       0       )
usage() {
 cat <<EOF
Usage: `basename $0` [options]
  -i  machine ID for starpu codelets
  -l  Use late submit of recv
  -m  List of matrices index
  -p  List of number of MPI process
  -t  List of number of threads
  -s  List of schedulers
  -r  Build directory
  -o  Output directory
  -h  Print the help
EOF
    echo ""
    echo "Matrices:"
    echo "__________________________"
    FORMAT="| %2s | %10s | %4s |\n"
    printf "$FORMAT" "ID" "MATRIX" "FACT"
    echo "|____|____________|______|"
    for i in `seq 0 $((${#matrices[*]} - 1))`
    do
	MATRIX=${matrices[$i]}
	MYPATH=${paths[$i]}
	BINARY=${binaries[$i]}
	DRIVER=${drivers[$i]}
	FACT=${facto[$i]}
	printf "$FORMAT" $i $MATRIX $FACT
    done
    echo "|________________________|"
    echo ""

    echo "Schedulers:"
    echo "____________________________________________________"
    FORMAT="| %2s | %17s | %10s | %10s |\n"
    printf "$FORMAT" "ID" "SCHEDULER" "FANIN" "NESTED"
    echo "|____|_____________________________________________|"
    for i in `seq 0 $((${#scheds[@]}-1))`
    do
        printf "$FORMAT" $i ${scheds[i]} ${fanin[i]} ${nested[i]}
    done
    echo "|__________________________________________________|"
    #echo "Schedulers:"
    #echo "__________________________"
    #FORMAT="| %2s | %17s |\n"
    #printf "$FORMAT" "ID" "SCHEDULER"
    #echo "|____|___________________|"
    #for i in `seq 0 $((${#scheds[*]} - 1))`
    #do
    #printf "$FORMAT" $i ${scheds[i]}
    #done
    #echo "|________________________|"
}
if [ $# -eq 0 ]
then
    usage
    exit 1
fi
while getopts ":c:i:lm:o:p:r:s:t:h" opt; do
    case $opt in
        c)
            CMIN=$OPTARG
            if [ $CMIN -eq "-1" ]
            then
                DEFAULT_ORDERING="API_YES"
            fi
            ;;
	i)
	    machine=$OPTARG
	    ;;
        l)
            LATE="API_YES"
            ;;
	m)
	    LISTMATRICES=$OPTARG
	    ;;
	o)
	    OUTPUTDIR=$OPTARG
	    ;;
	p)
	    LISTMPI=$OPTARG
	    ;;
	r)
	    ROOT=$OPTARG
	    ;;
	s)
	    LISTSCHED=$OPTARG
	    ;;
	t)
	    LISTCPU=$OPTARG
	    ;;
	h)
	    usage
	    exit 0
	    ;;
	\?)
	    echo "Invalid option: -$OPTARG" >&2
	    usage >&2
	    exit 1
	    ;;
	:)
	    echo "Option -$OPTARG requires an argument." >&2
	    usage >&2
	    exit 1
	    ;;
    esac
done
export EZTRACE=
mkdir -p ${OUTPUTDIR}/${date}
if [ -e ${ROOT}/CMakeCache.txt ]
then
    cp -r ${ROOT}/example/*_simple $OUTPUTDIR/${date}
    binaries=("${binaries_cmake[@]}")
    IPARM_DEFAULT_ORDERING="IPARM_ORDERING_DEFAULT"
    IPARM_ORDERING_CMIN="IPARM_SCOTCH_CMIN"
else
    cp -r ${ROOT}/example/bin/[sdcz]simple $OUTPUTDIR/${date}
    IPARM_DEFAULT_ORDERING="IPARM_DEFAULT_ORDERING"
    IPARM_ORDERING_CMIN="IPARM_ORDERING_CMIN"
fi
echo ls  $OUTPUTDIR/${date}
ls  $OUTPUTDIR/${date}
echo $LISTMPI
echo $LISTCPU
echo $LISTMATRICES
for nmpi in  $LISTMPI
do
    for ncpu in $LISTCPU
    do
	for i in $LISTMATRICES
	do
	    MATRIX=${matrices[$i]}
	    MYPATH=${paths[$i]}
	    BINARY=${binaries[$i]}
	    DRIVER=${drivers[$i]}
	    FACT=${facto[$i]}

	    #PASTIX_STARPU_FANIN=${fanin[$s]} \
	    #    PASTIX_STARPU_PREFETCH_ON_NODE=1 \
	    #    STARPU_GENERATE_TRACE=1 \
	    #    STARPU_SCHED_BETA=1.5 \
	    #    STARPU_SCHED=${SCHED}  \
	    #    ${MPIRUN} -pernode -np ${nmpi} $EZTRACE $OUTPUTDIR/${date}/${BINARY} -${DRIVER} ${MYPATH} \
	    #    -iparm IPARM_STARPU ${STARPU} -t ${ncpu} -v 4 \
	    #    -iparm IPARM_FACTORIZATION API_FACT_${FACT} \
	    #    -iparm IPARM_ORDERING_DEFAULT API_NO -iparm IPARM_SCOTCH_CMIN ${CMIN}  \
	    #    -iparm IPARM_GPU_CRITERIUM 2 \
	    #    -iparm IPARM_MIN_BLOCKSIZE ${MIN_BLOCKSIZE} \
	    #    -iparm IPARM_MAX_BLOCKSIZE ${MAX_BLOCKSIZE} \
	    #    -iparm IPARM_CUDA_NBR ${ncuda}  \
	    #    -iparm IPARM_IO_STRATEGY IPARM_IO_SAVE \
	    #    -iparm IPARM_END_TASK API_TASK_FAX 2>&1 | tee ${OUTPUT}/out

	    for s in $LISTSCHED
	    do
		SCHED=${scheds[$s]}
		MAXCUDA=${maxcuda[$s]}
		STARPU=${starpu[$s]}

		#for ncuda in 0 `seq $MAXCUDA`
		#do
		#ncores=112`cat /proc/cpuinfo | awk '/^processor/{print $3}' | tail -1`
		ncuda=$((8 - $ncpu))
		if [ $ncuda -gt $MAXCUDA ]
		then
		    ncuda=$MAXCUDA
                    continue
		fi
		MIN_BLOCKSIZE=60
		MAX_BLOCKSIZE=120
		OPTIONS=${SCHED}+${FACT}+5fact+flop+cmin${CMIN}+frat8+r11360+bsize${MIN_BLOCKSIZE}-${MAX_BLOCKSIZE}
		OUTPUT=$OUTPUTDIR/${machine}_${MATRIX}_${OPTIONS}_${nmpi}_${ncpu}_${ncuda}_FANIN${fanin[$s]}_NESTED${nested[$s]}_${rev}_${date}
		echo $OUTPUT
		mkdir -p $OUTPUT
		if [ -n "$EZTRACE" ]
		then
		    rm /tmp/${USER}_eztrace_log_rank_1
		fi
                if [ "$FRONTAL_NAME" = "curie" ]
                then

		    cat >> $OUTPUT/job.sh <<EOF
#!/bin/sh

#############################
# les directives PBS vont ici:

# Your job name (displayed by the queue)
#MSUB -r ${MATRIX}_${SCHED}_${fanin[$s]}
# Specify the working directory
#MSUB -e $OUTPUT/error.log
#MSUB -o $OUTPUT/output.log
#MSUB -q hybrid

# walltime 
#MSUB -T 3600

# Specify the number of nodes(nodes=) and the number of cores per nodes(ppn=) to be used
#MSUB -n $nmpi
#MSUB -N $nmpi
#MSUB -c $(($ncpu + $ncuda))
#MSUB -A gen1567

# fin des directives PBS
#############################
STARPU_HOME=starpu_home
cd $OUTPUTDIR

EOF

                else
		    cat >> $OUTPUT/job.sh <<EOF
#!/bin/sh

#############################
# les directives PBS vont ici:

# Your job name (displayed by the queue)
#PBS -N ${MATRIX}_${SCHED}_${fanin[$s]}

# Specify the working directory
#PBS -d $OUTPUT
#PBS -e $OUTPUT/error.log
#PBS -o $OUTPUT/output.log
#PBS -q $OUTPUT/output.log

# walltime (hh:mm::ss)
#PBS -l walltime=01:00:00

# Specify the number of nodes(nodes=) and the number of cores per nodes(ppn=) to be used
#PBS -l nodes=$nmpi:ppn=12

# fin des directives PBS
#############################
EOF
                fi
		cat >> $OUTPUT/job.sh <<EOF

# modules cleaning
module purge
source $ROOT/loadEnv
module add torque
module add mpiexec
module list

# useful informations to print
echo "#############################"
echo "User:" \$USER
echo "Date:" \`date\`
echo "Host:" \`hostname\`
echo "Directory:" \`pwd\`
echo "PBS_JOBID:" \$PBS_JOBID
echo "PBS_O_WORKDIR:" \$PBS_O_WORKDIR
echo "PBS_NODEFILE: " \`cat \$PBS_NODEFILE | uniq\`
echo "#############################"

cd $OUTPUTDIR
export STARPU_HOME=starpu_home
#echo $LD_LIBRARY_PATH
#export LD_LIBRARY_PATH=/opt/mpi/bullxmpi/1.2.7.2/bin:/opt/intel/14.0.3.174/bin/intel64:/ccc/cont003/home/gen1567/lacoste/work/git-install/bin:/ccc/cont003/home/gen1567/lacoste/gitflow:/usr/lib64/qt-3.3/bin:/usr/local/bin:/bin:/usr/bin:$LD_LIBRARY_PATH
#echo $LD_LIBRARY_PATH
# Chargement des modules :

${MPIRUN} uname -a

## export STARPU_HOSTNAME=$machine
export EZTRACE_FLUSH=1
export EZTRACE_TRACE=memory

export PASTIX_STARPU_FANIN=${fanin[$s]}
export PASTIX_STARPU_NESTED_TASK=${nested[$s]}
export PASTIX_STARPU_PREFETCH_ON_NODE=1
export STARPU_GENERATE_TRACE=1
export STARPU_SCHED_BETA=1.5
export STARPU_SCHED=${SCHED}
${MPIRUN} $EZTRACE $OUTPUTDIR/${date}/${BINARY} -${DRIVER} ${MYPATH} \\
     -iparm IPARM_STARPU ${STARPU} -t ${ncpu} -v 4 \\
     -iparm IPARM_FACTORIZATION API_FACT_${FACT} \\
     -iparm ${IPARM_DEFAULT_ORDERING} ${DEFAULT_ORDERING} \\
     -iparm ${IPARM_ORDERING_CMIN} ${CMIN}  \\
     -iparm IPARM_GPU_CRITERIUM 2 \\
     -iparm IPARM_MIN_BLOCKSIZE ${MIN_BLOCKSIZE} \\
     -iparm IPARM_MAX_BLOCKSIZE ${MAX_BLOCKSIZE} \\
     -iparm IPARM_CUDA_NBR ${ncuda} \\
     -iparm IPARM_STARPU_NESTED_TASKS ${nested[$s]} \\
     -iparm IPARM_STARPU_FANIN ${fanin[$s]} \\
     -iparm IPARM_STARPU_RECV_LATE $LATE
# all done
echo "Job finished"
EOF
                if [ "$FRONTAL_NAME" = "curie" ]
                then
                    ccc_msub $OUTPUT/job.sh
		    while [ $? -ne 0 ]
		    do
		        echo "failed ccc_msub $OUTPUT/job.sh"
		        echo `ccc_mstat -u | grep $USER | wc -l` " pending jobs"
		        echo "retrying in 60s..."
		        sleep 60
		        ccc_msub $OUTPUT/job.sh
		    done
                else
		    qsub $OUTPUT/job.sh
		    while [ $? -ne 0 ]
		    do
		        echo "failed qsub $OUTPUT/job.sh"
		        echo `qstat -u $USER | grep $USER | wc -l` pending jobs
		        echo "retrying in 60s..."
		        sleep 1
		        qsub $OUTPUT/job.sh
		    done
                fi
		echo "submit $OUTPUT/job.sh done"
	    done
	done
    done
done
