/* Copyright 2008 BORDEAUX I UNIVERSITY & INRIA 
**
** This file is part of the PaStiX parallel sparse matrix package.
**
** This software is governed by the CeCILL-C license under French law
** and abiding by the rules of distribution of free software. You can
** use, modify and/or redistribute the software under the terms of the
** CeCILL-C license as circulated by CEA, CNRS and INRIA at the following
** URL: "http://www.cecill.info".
** 
** As a counterpart to the access to the source code and rights to copy,
** modify and redistribute granted by the license, users are provided
** only with a limited warranty and the software's author, the holder of
** the economic rights, and the successive licensors have only limited
** liability.
** 
** In this respect, the user's attention is drawn to the risks associated
** with loading, using, modifying and/or developing or reproducing the
** software by the user in light of its specific status of free software,
** that may mean that it is complicated to manipulate, and that also
** therefore means that it is reserved for developers and experienced
** professionals having in-depth computer knowledge. Users are therefore
** encouraged to load and test the software's suitability as regards
** their requirements in conditions enabling the security of their
** systems and/or data to be ensured and, more generally, to use and
** operate it in the same conditions as regards security.
** 
** The fact that you are presently reading this means that you have had
** knowledge of the CeCILL-C license and that you accept its terms.
*/
/**
 * @file starpu_submit.c
 *
 * @author Xavier Lacoste
 * @precisions normal z -> s d c
 */

#include "starpu_defines.h"
#include "common_pastix.h"
#include "starpu_submit.h"
#include "starpu_kernels.h"
#include "sopalin_acces.h"
#include "core_zgeadd.h"
#include "perf.h"

/**
 * Computes the number of operation involved in a symmetric fanin addition task.
 *
 * @param task  The StarPU task.
 * @param arch  The StarPU starpu_perfmodel_archtype
 * @param nimpl Version of the executed implementation.
 *
 * @returns Number of operation involved in the addition.
 */
static size_t
starpu_syadd_size(struct starpu_task *task,
#ifdef STARPU_1_2
                   struct starpu_perfmodel_arch *arch,
#else
                   enum starpu_perfmodel_archtype arch,
#endif
                   unsigned nimpl) {
    Sopalin_Data_t    * sopalin_data;
    SolverCblk        * cblk, *fcblk;
    PASTIX_INT        stride;
    size_t              dima;
    starpu_codelet_unpack_args(task->cl_arg, &sopalin_data, &cblk, &fcblk);
    stride   = cblk->stride;
    dima     = cblk->lcolnum - cblk->fcolnum + 1;
    return OPS_GEAM(stride, dima);

}

/**
 * Computes the number of operation involved in an unsymmetric fanin addition
 * task.
 *
 * @param task  The StarPU task.
 * @param arch  The StarPU starpu_perfmodel_archtype
 * @param nimpl Version of the executed implementation.
 *
 * @returns Number of operation involved in the addition.
 */
static size_t
starpu_geadd_size(struct starpu_task *task,
#ifdef STARPU_1_2
                   struct starpu_perfmodel_arch *arch,
#else
                   enum starpu_perfmodel_archtype arch,
#endif
                   unsigned nimpl) {
    return 2*starpu_syadd_size(task, arch, nimpl);
}

/**
 * starpu_perfmodel assoctiated to the unsymmetric fanin addition task.
 */
static struct starpu_perfmodel starpu_geadd_model;

/**
 * starpu_perfmodel assoctiated to the symmetric fanin addition task.
 */
static struct starpu_perfmodel starpu_syadd_model;

/**
 * starpu_codelet describing the unsymmetric fanin addition task.
 */
#define starpu_geadd_cl PASTIX_PREFIX_F(starpu_geadd_cl)
struct starpu_codelet starpu_geadd_cl =
{
    .where = STARPU_CPU,
    .cpu_funcs[0] = starpu_zgetrfsp1d_geadd_cpu,
    //.cuda_funcs[0] = xxtrf_starpu_cuda,
    .model = &starpu_geadd_model,
    /* LU */
    .nbuffers = 5,
    .modes = {
        STARPU_R,
        STARPU_RW|STARPU_COMMUTE,
        STARPU_R,
        STARPU_RW|STARPU_COMMUTE,
        STARPU_R
    }
};

/**
 * starpu_codelet describing the symmetric fanin addition task.
 */
#define starpu_syadd_cl PASTIX_PREFIX_F(starpu_syadd_cl)
struct starpu_codelet starpu_syadd_cl =
{
    .where = STARPU_CPU,
    .cpu_funcs[0] = starpu_zsytrfsp1d_syadd_cpu,
    //.cuda_funcs[0] = xxtrf_starpu_cuda,
    .model = &starpu_syadd_model,
    /* LU */
    .nbuffers = 3,
    .modes = {
        STARPU_R,
        STARPU_RW|STARPU_COMMUTE,
        STARPU_R
    }
};

/**
 * Submition of one incomming fanin addition in the general case.
 *
 * @params sopalin_data Internal PaStiX structure.
 *
 * @returns PASTIX_SUCCESS if no error occurs.
 */
int
starpu_gesubmit_one_incomming_fanin( Sopalin_Data_t * sopalin_data, SolverCblk * target ) {
    SolverMatrix * datacode = sopalin_data->datacode;
    starpu_loop_data_t  *starpu_loop_data  = sopalin_data->starpu_loop_data;
    PASTIX_INT itertask, workerid = -1;
    starpu_data_handle_t *L_handle         = starpu_loop_data->L_handle;
    starpu_data_handle_t **Lfanin_handle   = starpu_loop_data->Lfanin_handle;
    starpu_data_handle_t *U_handle         = starpu_loop_data->U_handle;
    starpu_data_handle_t **Ufanin_handle   = starpu_loop_data->Ufanin_handle;
    PASTIX_INT max_deps = 0;
    int ret;
    SolverCblk *fanin;
    SolverBlok *blok;
    PASTIX_INT clustnum;

    starpu_geadd_model.type = STARPU_REGRESSION_BASED;
    starpu_geadd_model.symbol = "FANIN_ADD";
#ifdef STARPU_1_2
    starpu_perfmodel_init(&starpu_geadd_model);
//     starpu_geadd_model.per_arch[STARPU_CPU_WORKER][0][0][0].size_base  = starpu_geadd_size;
//     if(starpu_worker_get_count_by_type(STARPU_CUDA_WORKER) != 0)
//         starpu_geadd_model.per_arch[STARPU_CUDA_WORKER][0][0][0].size_base = starpu_geadd_size;
#  else
    starpu_geadd_model.per_arch[STARPU_CPU_WORKER][0].size_base  = starpu_geadd_size;
    starpu_geadd_model.per_arch[STARPU_CUDA_WORKER][0].size_base = starpu_geadd_size;
#  endif

    for (clustnum = 0; clustnum < datacode->clustnbr; clustnum++) {
        if (clustnum == datacode->clustnum) continue;
        for (fanin = datacode->fcblktab[clustnum];
             fanin < datacode->fcblktab[clustnum] + datacode->fcblknbr[clustnum];
             fanin++) {
          if (target->gcblknum != fanin->gcblknum) continue;
          else {
            SolverCblk  *fcblk;
            PASTIX_INT gfcblk = fanin->gcblknum;
            PASTIX_INT lfcblk = SOLV_GCBLK2LOC(gfcblk);
            PASTIX_INT faninnum = fcblk_getnum(datacode, fanin, clustnum);
            fcblk = datacode->cblktab+lfcblk;
            assert(cblk_islocal(datacode, fcblk) == API_YES);
            assert(lfcblk >= 0);
            assert(starpu_data_get_rank(Lfanin_handle[clustnum][faninnum]) == clustnum);
            assert(starpu_data_get_rank(Ufanin_handle[clustnum][faninnum]) == clustnum);
            ret =
                starpu_mpi_insert_task(sopalin_data->sopar->pastix_comm,
                                       &starpu_geadd_cl,
                                       STARPU_VALUE, &sopalin_data, sizeof(Sopalin_Data_t*),
                                       STARPU_VALUE, &fanin,        sizeof(SolverCblk*),
                                       STARPU_VALUE, &fcblk,        sizeof(SolverCblk*),
                                       STARPU_R,     Lfanin_handle[clustnum][faninnum],
                                       STARPU_COMMUTE|STARPU_RW,    L_handle[lfcblk],
                                       STARPU_R,     Ufanin_handle[clustnum][faninnum],
                                       STARPU_COMMUTE|STARPU_RW,    U_handle[lfcblk],
                                       STARPU_R, starpu_loop_data->blocktab_handles[SOLV_PROCNUM],
                                       STARPU_EXECUTE_ON_WORKER, workerid,
                                       STARPU_CALLBACK,     starpu_prof_callback,
                                       STARPU_CALLBACK_ARG, sopalin_data->hgemm_stats,
#ifdef STARPU_CONTEXT
                                       STARPU_SCHED_CTX, sched_ctxs[my_ctx],
#endif
                                       STARPU_PRIORITY,  INT_MAX,
                                       0);
            STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_insert_task");
            sopalin_data->tasknbr++;
            sopalin_data->task_incomming_fanin++;
            starpu_mpi_cache_flush(sopalin_data->sopar->pastix_comm,
                                   Lfanin_handle[clustnum][faninnum]);

          }
        }
    }
    return PASTIX_SUCCESS;
}

/**
 * Submition of all incomming fanin addition in the general case.
 *
 * @params sopalin_data Internal PaStiX structure.
 *
 * @returns PASTIX_SUCCESS if no error occurs.
 */
int
starpu_gesubmit_incomming_fanin( Sopalin_Data_t * sopalin_data ) {
    SolverMatrix * datacode = sopalin_data->datacode;
    starpu_loop_data_t  *starpu_loop_data  = sopalin_data->starpu_loop_data;
    PASTIX_INT itertask, workerid = -1;
    starpu_data_handle_t *L_handle         = starpu_loop_data->L_handle;
    starpu_data_handle_t **Lfanin_handle   = starpu_loop_data->Lfanin_handle;
    starpu_data_handle_t *U_handle         = starpu_loop_data->U_handle;
    starpu_data_handle_t **Ufanin_handle   = starpu_loop_data->Ufanin_handle;
    PASTIX_INT max_deps = 0;
    int ret;
    SolverCblk *fanin;
    SolverBlok *blok;
    PASTIX_INT clustnum;

    starpu_geadd_model.type = STARPU_REGRESSION_BASED;
    starpu_geadd_model.symbol = "FANIN_ADD";
#ifdef STARPU_1_2
    starpu_perfmodel_init(&starpu_geadd_model);
//     starpu_geadd_model.per_arch[STARPU_CPU_WORKER][0][0][0].size_base  = starpu_geadd_size;
//     if(starpu_worker_get_count_by_type(STARPU_CUDA_WORKER) != 0)
//         starpu_geadd_model.per_arch[STARPU_CUDA_WORKER][0][0][0].size_base = starpu_geadd_size;
#  else
    starpu_geadd_model.per_arch[STARPU_CPU_WORKER][0].size_base  = starpu_geadd_size;
    starpu_geadd_model.per_arch[STARPU_CUDA_WORKER][0].size_base = starpu_geadd_size;
#  endif

    for (clustnum = 0; clustnum < datacode->clustnbr; clustnum++) {
        if (clustnum == datacode->clustnum) continue;
        for (fanin = datacode->fcblktab[clustnum];
             fanin < datacode->fcblktab[clustnum] + datacode->fcblknbr[clustnum];
             fanin++) {
            SolverCblk  *fcblk;
            PASTIX_INT gfcblk = fanin->gcblknum;
            PASTIX_INT lfcblk = SOLV_GCBLK2LOC(gfcblk);
            PASTIX_INT faninnum = fcblk_getnum(datacode, fanin, clustnum);
            fcblk = datacode->cblktab+lfcblk;
            assert(cblk_islocal(datacode, fcblk) == API_YES);
            assert(lfcblk >= 0);
            assert(starpu_data_get_rank(Lfanin_handle[clustnum][faninnum]) == clustnum);
            assert(starpu_data_get_rank(Ufanin_handle[clustnum][faninnum]) == clustnum);
            ret =
                starpu_mpi_insert_task(sopalin_data->sopar->pastix_comm,
                                       &starpu_geadd_cl,
                                       STARPU_VALUE, &sopalin_data, sizeof(Sopalin_Data_t*),
                                       STARPU_VALUE, &fanin,        sizeof(SolverCblk*),
                                       STARPU_VALUE, &fcblk,        sizeof(SolverCblk*),
                                       STARPU_R,     Lfanin_handle[clustnum][faninnum],
                                       STARPU_COMMUTE|STARPU_RW,    L_handle[lfcblk],
                                       STARPU_R,     Ufanin_handle[clustnum][faninnum],
                                       STARPU_COMMUTE|STARPU_RW,    U_handle[lfcblk],
                                       STARPU_R, starpu_loop_data->blocktab_handles[SOLV_PROCNUM],
                                       STARPU_EXECUTE_ON_WORKER, workerid,
                                       STARPU_CALLBACK,     starpu_prof_callback,
                                       STARPU_CALLBACK_ARG, sopalin_data->hgemm_stats,
#ifdef STARPU_CONTEXT
                                       STARPU_SCHED_CTX, sched_ctxs[my_ctx],
#endif
                                       STARPU_PRIORITY,  INT_MAX,
                                       0);
            STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_insert_task");
            sopalin_data->tasknbr++;
            sopalin_data->task_incomming_fanin++;
        }
    }
    return PASTIX_SUCCESS;
}


/**
 * Submition of one incomming fanin addition in the symmetric case.
 *
 * @params sopalin_data Internal PaStiX structure.
 *
 * @returns PASTIX_SUCCESS if no error occurs.
 */
int
starpu_sysubmit_one_incomming_fanin( Sopalin_Data_t * sopalin_data, SolverCblk * target ) {
    SolverMatrix * datacode = sopalin_data->datacode;
    starpu_loop_data_t  *starpu_loop_data  = sopalin_data->starpu_loop_data;
    PASTIX_INT itertask, workerid = -1;
    starpu_data_handle_t *L_handle         = starpu_loop_data->L_handle;
    starpu_data_handle_t **Lfanin_handle   = starpu_loop_data->Lfanin_handle;
    PASTIX_INT max_deps = 0;
    PASTIX_INT clustnum;
    int ret;
    SolverCblk *fanin;
    SolverBlok *blok;

    starpu_syadd_model.type = STARPU_REGRESSION_BASED;
    starpu_syadd_model.symbol = "FANIN_ADD";
#ifdef STARPU_1_2
    starpu_perfmodel_init(&starpu_syadd_model);
//     starpu_syadd_model.per_arch[STARPU_CPU_WORKER][0][0][0].size_base  = starpu_syadd_size;
//     if(starpu_worker_get_count_by_type(STARPU_CUDA_WORKER) != 0)
//         starpu_syadd_model.per_arch[STARPU_CUDA_WORKER][0][0][0].size_base = starpu_syadd_size;
#  else
    starpu_syadd_model.per_arch[STARPU_CPU_WORKER][0].size_base  = starpu_syadd_size;
    starpu_syadd_model.per_arch[STARPU_CUDA_WORKER][0].size_base = starpu_syadd_size;
#  endif

    for (clustnum = 0; clustnum < datacode->clustnbr; clustnum++) {
        if (clustnum == datacode->clustnum) continue;
        for (fanin = datacode->fcblktab[clustnum];
             fanin < datacode->fcblktab[clustnum] + datacode->fcblknbr[clustnum];
             fanin++) {
          if (fanin->gcblknum != target->gcblknum) continue;
          else {
            SolverCblk *fcblk;
            PASTIX_INT gfcblk = fanin->gcblknum;
            PASTIX_INT lfcblk = SOLV_GCBLK2LOC(gfcblk);
            PASTIX_INT faninnum = fcblk_getnum(datacode, fanin, clustnum);
            fcblk = datacode->cblktab+lfcblk;

            assert(lfcblk >= 0);
            assert(cblk_islocal(datacode, fcblk));
            ret =
                starpu_mpi_insert_task(sopalin_data->sopar->pastix_comm,
                                       &starpu_syadd_cl,
                                       STARPU_VALUE, &sopalin_data, sizeof(Sopalin_Data_t*),
                                       STARPU_VALUE, &fanin,        sizeof(SolverCblk*),
                                       STARPU_VALUE, &fcblk,       sizeof(SolverCblk*),
                                       STARPU_R,     Lfanin_handle[clustnum][faninnum],
                                       STARPU_COMMUTE|STARPU_RW,    L_handle[lfcblk],
                                       STARPU_R, starpu_loop_data->blocktab_handles[SOLV_PROCNUM],
#ifdef STARPU_1_2
                                       STARPU_EXECUTE_ON_WORKER, workerid,
#endif
                                       STARPU_CALLBACK,     starpu_prof_callback,
                                       STARPU_CALLBACK_ARG, sopalin_data->hgemm_stats,
#ifdef STARPU_CONTEXT
                                       STARPU_SCHED_CTX, sched_ctxs[my_ctx],
#endif
                                       STARPU_PRIORITY,  INT_MAX,
                                       0);
            STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_insert_task");
            sopalin_data->tasknbr++;
            sopalin_data->task_incomming_fanin++;
            starpu_mpi_cache_flush(sopalin_data->sopar->pastix_comm,
                                   Lfanin_handle[clustnum][faninnum]);
          }
        }
    }
    return PASTIX_SUCCESS;
  return NO_ERR;
}
/**
 * Submition of all incomming fanin addition in the symmetric case.
 *
 * @params sopalin_data Internal PaStiX structure.
 *
 * @returns PASTIX_SUCCESS if no error occurs.
 */
int
starpu_sysubmit_incomming_fanin( Sopalin_Data_t * sopalin_data ) {
    SolverMatrix * datacode = sopalin_data->datacode;
    starpu_loop_data_t  *starpu_loop_data  = sopalin_data->starpu_loop_data;
    PASTIX_INT itertask, workerid = -1;
    starpu_data_handle_t *L_handle         = starpu_loop_data->L_handle;
    starpu_data_handle_t **Lfanin_handle   = starpu_loop_data->Lfanin_handle;
    PASTIX_INT max_deps = 0;
    PASTIX_INT clustnum;
    int ret;
    SolverCblk *fanin;
    SolverBlok *blok;

    starpu_syadd_model.type = STARPU_REGRESSION_BASED;
    starpu_syadd_model.symbol = "FANIN_ADD";
#ifdef STARPU_1_2
    starpu_perfmodel_init(&starpu_syadd_model);
//     starpu_syadd_model.per_arch[STARPU_CPU_WORKER][0][0][0].size_base  = starpu_syadd_size;
//     if(starpu_worker_get_count_by_type(STARPU_CUDA_WORKER) != 0)
//         starpu_syadd_model.per_arch[STARPU_CUDA_WORKER][0][0][0].size_base = starpu_syadd_size;
#  else
    starpu_syadd_model.per_arch[STARPU_CPU_WORKER][0].size_base  = starpu_syadd_size;
    starpu_syadd_model.per_arch[STARPU_CUDA_WORKER][0].size_base = starpu_syadd_size;
#  endif

    for (clustnum = 0; clustnum < datacode->clustnbr; clustnum++) {
        if (clustnum == datacode->clustnum) continue;
        for (fanin = datacode->fcblktab[clustnum];
             fanin < datacode->fcblktab[clustnum] + datacode->fcblknbr[clustnum];
             fanin++) {
            SolverCblk *fcblk;
            PASTIX_INT gfcblk = fanin->gcblknum;
            PASTIX_INT lfcblk = SOLV_GCBLK2LOC(gfcblk);
            PASTIX_INT faninnum = fcblk_getnum(datacode, fanin, clustnum);
            fcblk = datacode->cblktab+lfcblk;

            assert(lfcblk >= 0);
            assert(cblk_islocal(datacode, fcblk));
            ret =
                starpu_mpi_insert_task(sopalin_data->sopar->pastix_comm,
                                       &starpu_syadd_cl,
                                       STARPU_VALUE, &sopalin_data, sizeof(Sopalin_Data_t*),
                                       STARPU_VALUE, &fanin,        sizeof(SolverCblk*),
                                       STARPU_VALUE, &fcblk,       sizeof(SolverCblk*),
                                       STARPU_R,     Lfanin_handle[clustnum][faninnum],
                                       STARPU_COMMUTE|STARPU_RW,    L_handle[lfcblk],
                                       STARPU_R, starpu_loop_data->blocktab_handles[SOLV_PROCNUM],
#ifdef STARPU_1_2
                                       STARPU_EXECUTE_ON_WORKER, workerid,
#endif
                                       STARPU_CALLBACK,     starpu_prof_callback,
                                       STARPU_CALLBACK_ARG, sopalin_data->hgemm_stats,
#ifdef STARPU_CONTEXT
                                       STARPU_SCHED_CTX, sched_ctxs[my_ctx],
#endif
                                       STARPU_PRIORITY,  INT_MAX,
                                       0);
            STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_insert_task");
            sopalin_data->tasknbr++;
            sopalin_data->task_incomming_fanin++;
        }
    }
    return PASTIX_SUCCESS;
}

/**
 * Submit one outgoing fanin update in the general case.
 *
 * @params sopalin_data Internal PaStiX structure.
 * @params fcblk Fanin column block that will update a remote column block.
 * @params hcblk Halo column block that will be updated.
 *
 * @returns PASTIX_SUCCESS if no error occurs.
 */
int
starpu_gesubmit_outgoing_fanin( Sopalin_Data_t * sopalin_data,
                                 SolverCblk     * fcblk,
                                 SolverCblk     * hcblk ) {
    SolverMatrix          *datacode         = sopalin_data->datacode;
    starpu_loop_data_t    *starpu_loop_data = sopalin_data->starpu_loop_data;
    starpu_data_handle_t  *Lhalo_handle     = starpu_loop_data->Lhalo_handle;
    starpu_data_handle_t  *Lfanin_handle    = starpu_loop_data->Lfanin_handle[SOLV_PROCNUM];
    starpu_data_handle_t  *Uhalo_handle     = starpu_loop_data->Uhalo_handle;
    starpu_data_handle_t  *Ufanin_handle    = starpu_loop_data->Ufanin_handle[SOLV_PROCNUM];
    PASTIX_INT fcblkidx   = fcblk_getnum(datacode, fcblk, SOLV_PROCNUM);
    PASTIX_INT hcblkidx   = hcblk_getnum(datacode, hcblk);
    int ret, workerid = -1;
#ifdef STARPU_CONTEXT
    PASTIX_INT sched_ctxs = starpu_loop_data->sched_ctxs;
    PASTIX_INT threadid = TASK_THREADID(itertask);
    PASTIX_INT thread_per_ctx = starpu_loop_data->thread_per_ctx;
    PASTIX_INT my_ctx;
    my_ctx = 1+threadid/thread_per_ctx;
#endif
    assert(hcblk->procdiag  == starpu_data_get_rank(Lhalo_handle[hcblkidx]));
    assert(hcblk->procdiag < datacode->clustnbr);
    ret =
        starpu_mpi_insert_task(sopalin_data->sopar->pastix_comm,
                               &starpu_geadd_cl,
                               STARPU_VALUE, &sopalin_data, sizeof(Sopalin_Data_t*),
                               STARPU_VALUE, &fcblk,        sizeof(SolverCblk*),
                               STARPU_VALUE, &hcblk,        sizeof(SolverCblk*),
                               STARPU_R,                    Lfanin_handle[fcblkidx],
                               STARPU_COMMUTE|STARPU_RW,    Lhalo_handle[hcblkidx],
                               STARPU_R,                    Ufanin_handle[fcblkidx],
                               STARPU_COMMUTE|STARPU_RW,    Uhalo_handle[hcblkidx],
                               STARPU_R, starpu_loop_data->blocktab_handles[hcblk->procdiag],
#ifdef STARPU_1_1
                               STARPU_EXECUTE_ON_WORKER, workerid,
#endif
                               STARPU_CALLBACK,     starpu_prof_callback,
                               STARPU_CALLBACK_ARG, sopalin_data->hgemm_stats,
#ifdef STARPU_CONTEXT
                               STARPU_SCHED_CTX,    sched_ctxs[my_ctx],
#endif
                               STARPU_PRIORITY,  INT_MAX,
                              0);
    STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_insert_task");
    sopalin_data->tasknbr++;
    sopalin_data->task_outgoing_fanin++;

    starpu_mpi_cache_flush(sopalin_data->sopar->pastix_comm,
                           Lfanin_handle[fcblkidx]);
    starpu_mpi_cache_flush(sopalin_data->sopar->pastix_comm,
                           Ufanin_handle[fcblkidx]);
    starpu_data_unregister_submit(Lfanin_handle[fcblkidx]);
    starpu_data_unregister_submit(Ufanin_handle[fcblkidx]);
    return PASTIX_SUCCESS;
}

/**
 * Submit one outgoing fanin update in the symmetric case.
 *
 * @params sopalin_data Internal PaStiX structure.
 * @params fcblk Fanin column block that will update a remote column block.
 * @params hcblk Halo column block that will be updated.
 *
 * @returns PASTIX_SUCCESS if no error occurs.
 */
int
starpu_sysubmit_outgoing_fanin( Sopalin_Data_t * sopalin_data,
                                 SolverCblk     * fcblk,
                                 SolverCblk     * hcblk ) {
    SolverMatrix          *datacode         = sopalin_data->datacode;
    starpu_loop_data_t    *starpu_loop_data = sopalin_data->starpu_loop_data;
    starpu_data_handle_t  *Lhalo_handle     = starpu_loop_data->Lhalo_handle;
    starpu_data_handle_t  *Lfanin_handle    = starpu_loop_data->Lfanin_handle[SOLV_PROCNUM];
    PASTIX_INT fcblkidx   = fcblk_getnum(datacode, fcblk, SOLV_PROCNUM);
    PASTIX_INT hcblkidx   = hcblk_getnum(datacode, hcblk);
    int ret, workerid = -1;
#ifdef STARPU_CONTEXT
    PASTIX_INT sched_ctxs = starpu_loop_data->sched_ctxs;
    PASTIX_INT threadid = TASK_THREADID(itertask);
    PASTIX_INT thread_per_ctx = starpu_loop_data->thread_per_ctx;
    PASTIX_INT my_ctx;
    my_ctx = 1+threadid/thread_per_ctx;
#endif
    assert(hcblk->procdiag  == starpu_data_get_rank(Lhalo_handle[hcblkidx]));
    assert(hcblk->procdiag < datacode->clustnbr);
    assert(fcblk->procdiag  == hcblk->procdiag);
    assert(fcblk->procdiag < datacode->clustnbr);
    assert(datacode->clustnum == starpu_data_get_rank(Lfanin_handle[fcblkidx]));
    ret =
        starpu_mpi_insert_task(sopalin_data->sopar->pastix_comm,
                               &starpu_syadd_cl,
                               STARPU_VALUE, &sopalin_data, sizeof(Sopalin_Data_t*),
                               STARPU_VALUE, &fcblk,        sizeof(SolverCblk*),
                               STARPU_VALUE, &hcblk,        sizeof(SolverCblk*),
                               STARPU_R,                    Lfanin_handle[fcblkidx],
                               STARPU_COMMUTE|STARPU_RW,    Lhalo_handle[hcblkidx],
                               STARPU_R, starpu_loop_data->blocktab_handles[hcblk->procdiag],
#ifdef STARPU_1_1
                               STARPU_EXECUTE_ON_WORKER, workerid,
#endif
                               STARPU_CALLBACK,     starpu_prof_callback,
                               STARPU_CALLBACK_ARG, sopalin_data->hgemm_stats,
#ifdef STARPU_CONTEXT
                               STARPU_SCHED_CTX,    sched_ctxs[my_ctx],
#endif
                               STARPU_PRIORITY,  INT_MAX,
                               0);
    STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_insert_task");
    sopalin_data->tasknbr++;
    sopalin_data->task_outgoing_fanin++;
    starpu_mpi_cache_flush(sopalin_data->sopar->pastix_comm,
                           Lfanin_handle[fcblkidx]);
    starpu_data_unregister_submit(Lfanin_handle[fcblkidx]);
    return PASTIX_SUCCESS;
}
