/*@@
   @file      pGF_PostSend.c
   @date      Thu Apr  3 11:58:14 1997
   @author    Paul Walker
   @desc 
   The send-arm of the three-way pugh communcations branch
   which is @seefile pGF_PostRecv.v, @seefile pGF_PostSend.c
   and @seefile pGF_FinishRecv.c. See the documentation for
   @seeroutine pGF_PostSend for details on this routine.
   @enddesc 
   @version $Header$
 @@*/

#include <stdio.h>

#include "cctk.h"

#include "pugh.h"

static char *rcsid = "$Header$";

void pGF_PostSend1(pGH *GH, pGF *GF, int dir);
void pGF_PostSend3(pGH *GH, pGF *GF, int dir);

/*#define DEBUG_PRINT*/

 /*@@
   @routine    pGF_PostSend
   @date       Thu Apr  3 11:58:59 1997
   @author     Paul Walker
   @desc 
   This routine posts an asyncronous MPI send of a buffer
   for a face (MPI_Isend). It does this by first packing up
   a send buffer (allocated in @seeroutine SetupPGF) then
   sending it out on the pipe.
   <p>
   Since this is an asynchronous communications model we
   assume that a recieve has been posted for the send.
   thus the correct calling order for this is as in
   @seeroutine SyncGF, eg, after a Recv.
   <p>
   Note this does <b>not</b> wait on the send buffer from previous
   communications. It is the users responsibility to wait on
   that buffer.
   @enddesc 
   @calledby   SyncGroupGF
   @history
   @hdate Nov 4 1998 @hauthor Gabrielle Allen
   @hdesc Allow for forced synchronization of all GFs with storage
   @endhistory
@@*/

void pGF_PostSend(pGH *GH, pGF *GF, int dir)
{
#ifdef MPI
  int stag, dircomp;
  double ts, tw, tp;
  MPI_Datatype mpi_type;
  MPI_Datatype *send_dt;
  MPI_Status ms;

  /* Return if GF has no storage */
  if (!(GF->storage)) return;

  /* Return if communication not required and no forced synchronisation */
  if (!(GH->forceSync || GF->docomm[dir])) return;

  if (GH->neighbors[GH->myproc][dir] < 0) return;

  switch (GF->vtype) 
  {
    case CCTK_VARIABLE_CHAR:
      mpi_type = PUGH_MPI_CHAR;
      send_dt = GH->send_char_dt [GF->stagger];
      break;

    case CCTK_VARIABLE_INT:
      mpi_type = PUGH_MPI_INT;
      send_dt = GH->send_int_dt [GF->stagger];
      break;

    case CCTK_VARIABLE_REAL:
      mpi_type = PUGH_MPI_REAL;
      send_dt = GH->send_real_dt [GF->stagger];
      break;

    case CCTK_VARIABLE_COMPLEX:
      mpi_type = GH->pugh_mpi_complex;
      send_dt = GH->send_complex_dt [GF->stagger];
      break;

    default:
      CCTK_WARN (1, "Unknown variable type in pGF_PostRecv");
      return;
  }

  ts = MPI_Wtime();
  dircomp = dir+1;                /* Complementary direction */
  if (dircomp %2 == 0) dircomp = dir-1;
  /* Note this is the complement of the rtag set in PostRecv */
  stag = 1000 + dircomp + 2 * (GH->neighbors[GH->myproc][dir] + 
                             GH->nprocs * GF->gfno);
  /* mod the tag to force MPI compliance */
  stag = stag % 32768;

#ifdef DEBUG_PRINT
  printf ("SEND: GF %s Side %d Proc %d stag %d Size %d to %d\n",
          GF->name,dir,GH->myproc,stag,
          GF->buffer_sz[dir],GH->neighbors[GH->myproc][dir]);
#endif

  if (GH->commmodel == PUGH_DERIVEDTYPES)
  {
    CACTUS_MPI_ERROR (MPI_Isend (GF->data,
                      1, send_dt [dir],
                      GH->neighbors [GH->myproc][dir], stag,
                      GH->PUGH_COMM_WORLD, &(GF->sreq [dir])));
  }

  if (GH->commmodel == PUGH_ALLOCATEDBUFFERS) 
  {
    if (GH->dim == 3)
    {
      pGF_PostSend3(GH,GF,dir);
    }
    else if (GH->dim == 1)
    {
      pGF_PostSend1(GH,GF,dir);
    }
    else
    {
      CCTK_WARN(0,"PUGH does not yet support this dimension");
    }

    tp = MPI_Wtime();
    
    /* post send */
    CACTUS_MPI_ERROR (MPI_Isend (GF->send_buffer [dir],
                      GF->buffer_sz [dir], mpi_type,
                      GH->neighbors [GH->myproc][dir], stag,
                      GH->PUGH_COMM_WORLD, &(GF->sreq [dir])));
    tw = MPI_Wtime();
#ifdef COMM_TIMING
    printf ("GF %s Dir %d Time %f\n", GF->name, dir, tw-ts);
#endif
  }
#endif
}

void pGF_PostSend3(pGH *GH, pGF *GF, int dir)
{

  int ii,jj,kk,xx;
  int istart, iend;
  int jstart, jend;
  int kstart, kend;
  CCTK_CHAR *char_data;
  CCTK_INT *int_data;
  CCTK_REAL *real_data;
  CCTK_COMPLEX *complex_data;

  /* Copy my information into the send buffer */
  istart = GH->overlap[GF->stagger][0][dir][0];
  iend   = GH->overlap[GF->stagger][1][dir][0];
  jstart = GH->overlap[GF->stagger][0][dir][1];
  jend   = GH->overlap[GF->stagger][1][dir][1];
  kstart = GH->overlap[GF->stagger][0][dir][2];
  kend   = GH->overlap[GF->stagger][1][dir][2];
    
  /* Great now copy */
  xx = 0;
  switch (GF->vtype) 
  {
    case CCTK_VARIABLE_CHAR:
      char_data = (CCTK_CHAR *) GF->send_buffer [dir];
      for (kk=kstart; kk<kend; kk++)
      {
        for (jj=jstart; jj<jend; jj++)
        {
          for (ii=istart; ii<iend; ii++)
          {
            char_data [xx++] =
              ((CCTK_CHAR *) GF->data) [DATINDEX (GH,ii,jj,kk)];
          }
        }
      }
      break;
    
    case CCTK_VARIABLE_INT:
      int_data = (CCTK_INT *) GF->send_buffer [dir];
      for (kk=kstart; kk<kend; kk++)
      { 
        for (jj=jstart; jj<jend; jj++)
        {
          for (ii=istart; ii<iend; ii++)
          {
            int_data [xx++] =
              ((CCTK_INT *) GF->data) [DATINDEX (GH,ii,jj,kk)];
          }
        }
      }
      break;
    
    case CCTK_VARIABLE_REAL:
      real_data = (CCTK_REAL *) GF->send_buffer [dir];
      for (kk=kstart; kk<kend; kk++)
      { 
        for (jj=jstart; jj<jend; jj++)
        {
          for (ii=istart; ii<iend; ii++)
          {
            real_data [xx++] =
              ((CCTK_REAL *) GF->data) [DATINDEX (GH,ii,jj,kk)];
          }
        }
      }
      break;
    
    case CCTK_VARIABLE_COMPLEX:
      complex_data = (CCTK_COMPLEX *) GF->send_buffer [dir];
      for (kk=kstart; kk<kend; kk++)
      { 
        for (jj=jstart; jj<jend; jj++)
        {
          for (ii=istart; ii<iend; ii++)
          {
            complex_data [xx++] =
              ((CCTK_COMPLEX *) GF->data) [DATINDEX (GH,ii,jj,kk)];
          }
        }
      }
      break;
  }
}



void pGF_PostSend1(pGH *GH, pGF *GF, int dir)
{

  int ii,xx;
  int istart, iend;
  CCTK_CHAR *char_data;
  CCTK_INT *int_data;
  CCTK_REAL *real_data;
  CCTK_COMPLEX *complex_data;

  /* Copy my information into the send buffer */
  istart = GH->overlap[GF->stagger][0][dir][0];
  iend   = GH->overlap[GF->stagger][1][dir][0];
    
  /* Great now copy */
  xx = 0;
  switch (GF->vtype) 
  {
    case CCTK_VARIABLE_CHAR:
      char_data = (CCTK_CHAR *) GF->send_buffer [dir];
      for (ii=istart; ii<iend; ii++)
      {
        char_data [xx++] =
          ((CCTK_CHAR *) GF->data) [ii];
      }
      break;
    
    case CCTK_VARIABLE_INT:
      int_data = (CCTK_INT *) GF->send_buffer [dir];
      for (ii=istart; ii<iend; ii++)
      {
        int_data [xx++] =
          ((CCTK_INT *) GF->data) [ii];
      }
      break;
    
    case CCTK_VARIABLE_REAL:
      real_data = (CCTK_REAL *) GF->send_buffer [dir];
      for (ii=istart; ii<iend; ii++)
      {
        real_data [xx++] =
          ((CCTK_REAL *) GF->data) [ii];
      }
      break;
    
    case CCTK_VARIABLE_COMPLEX:
      complex_data = (CCTK_COMPLEX *) GF->send_buffer [dir];
      for (ii=istart; ii<iend; ii++)
      {
        complex_data [xx++] =
          ((CCTK_COMPLEX *) GF->data) [ii];
      }
      break;
  }
}