transfer: implement and use 1D interpolation

Stop abusing "2D of y size 1" for this.

1 files changed, 107 insertions, 49 deletions

diff --git a/transfer.c b/transfer.c
index 9c5e125..ee635b9 100644
--- a/transfer.c
+++ b/transfer.c
@@ -41,30 +41,35 @@ typedef struct GridTransfer {
 typedef struct GridTransferLagrange {
     unsigned int stencil;
 
-    void (*transfer_cont)   (double *dst, ptrdiff_t dst_len,
-                             const double *src, ptrdiff_t src_stride,
-                             const ptrdiff_t *idx_x, const double *fact_x, const double *fact_y);
-    void (*transfer_generic)(double *dst, ptrdiff_t dst_len,
-                             const double *src, ptrdiff_t src_stride,
-                             const ptrdiff_t *idx_x, const double *fact_x, const double *fact_y,
-                             ptrdiff_t dst_stride0, ptrdiff_t src_stride0);
+    void (*transfer2d_cont)   (double *dst, ptrdiff_t dst_len,
+                               const double *src, ptrdiff_t src_stride,
+                               const ptrdiff_t *idx_x, const double *fact_x, const double *fact_y);
+    void (*transfer2d_generic)(double *dst, ptrdiff_t dst_len,
+                               const double *src, ptrdiff_t src_stride,
+                               const ptrdiff_t *idx_x, const double *fact_x, const double *fact_y,
+                               ptrdiff_t dst_stride0, ptrdiff_t src_stride0);
+    void (*transfer1d_cont)   (double *dst, ptrdiff_t dst_len,
+                               const double *src, const ptrdiff_t *idx_x, const double *fact_x);
+    void (*transfer1d_generic)(double *dst, ptrdiff_t dst_len,
+                               const double *src, const ptrdiff_t *idx_x, const double *fact_x,
+                               ptrdiff_t dst_stride0, ptrdiff_t src_stride0);
 
     const NDArray *src;
     NDArray       *dst;
 
-    ptrdiff_t *idx[2];
-    double    *fact[2];
+    ptrdiff_t **idx;
+    double    **fact;
 } GridTransferLagrange;
 
 #if HAVE_EXTERNAL_ASM
-void mg2di_transfer_interp_line_cont_4_fma3(double *dst, ptrdiff_t dst_len,
-                                            const double *src, ptrdiff_t src_stride,
-                                            const ptrdiff_t *idx_x,
-                                            const double *fact_x, const double *fact_y);
-void mg2di_transfer_interp_line_cont_6_fma3(double *dst, ptrdiff_t dst_len,
-                                            const double *src, ptrdiff_t src_stride,
-                                            const ptrdiff_t *idx_x,
-                                            const double *fact_x, const double *fact_y);
+void mg2di_transfer_interp2d_line_cont_4_fma3(double *dst, ptrdiff_t dst_len,
+                                              const double *src, ptrdiff_t src_stride,
+                                              const ptrdiff_t *idx_x,
+                                              const double *fact_x, const double *fact_y);
+void mg2di_transfer_interp2d_line_cont_6_fma3(double *dst, ptrdiff_t dst_len,
+                                              const double *src, ptrdiff_t src_stride,
+                                              const ptrdiff_t *idx_x,
+                                              const double *fact_x, const double *fact_y);
 #endif
 
 #define STENCIL 2
@@ -153,43 +158,59 @@ static int transfer_lagrange_init(GridTransferContext *ctx)
     GridTransferLagrange *priv = ctx->priv;
     int ret;
 
+    if (ctx->nb_dims > 2)
+        return -ENOSYS;
+
     switch (ctx->op) {
     case GRID_TRANSFER_LAGRANGE_1:
         priv->stencil = 2;
-        priv->transfer_cont    = interp_transfer_line_cont_2;
-        priv->transfer_generic = interp_transfer_line_generic_2;
+        priv->transfer1d_cont    = interp1d_transfer_line_cont_2;
+        priv->transfer1d_generic = interp1d_transfer_line_generic_2;
+        priv->transfer2d_cont    = interp2d_transfer_line_cont_2;
+        priv->transfer2d_generic = interp2d_transfer_line_generic_2;
         break;
     case GRID_TRANSFER_LAGRANGE_3:
-        priv->transfer_cont    = interp_transfer_line_cont_4;
-        priv->transfer_generic = interp_transfer_line_generic_4;
+        priv->transfer1d_cont    = interp1d_transfer_line_cont_4;
+        priv->transfer1d_generic = interp1d_transfer_line_generic_4;
+        priv->transfer2d_cont    = interp2d_transfer_line_cont_4;
+        priv->transfer2d_generic = interp2d_transfer_line_generic_4;
         priv->stencil = 4;
 
 #if HAVE_EXTERNAL_ASM
         if (ctx->cpuflags & MG2DI_CPU_FLAG_FMA3) {
-            priv->transfer_cont    = mg2di_transfer_interp_line_cont_4_fma3;
+            priv->transfer2d_cont    = mg2di_transfer_interp2d_line_cont_4_fma3;
         }
 #endif
         break;
     case GRID_TRANSFER_LAGRANGE_5:
-        priv->transfer_cont    = interp_transfer_line_cont_6;
-        priv->transfer_generic = interp_transfer_line_generic_6;
+        priv->transfer1d_cont    = interp1d_transfer_line_cont_6;
+        priv->transfer1d_generic = interp1d_transfer_line_generic_6;
+        priv->transfer2d_cont    = interp2d_transfer_line_cont_6;
+        priv->transfer2d_generic = interp2d_transfer_line_generic_6;
         priv->stencil = 6;
 #if HAVE_EXTERNAL_ASM
         if (ctx->cpuflags & MG2DI_CPU_FLAG_FMA3) {
-            priv->transfer_cont    = mg2di_transfer_interp_line_cont_6_fma3;
+            priv->transfer2d_cont    = mg2di_transfer_interp2d_line_cont_6_fma3;
         }
 #endif
         break;
     case GRID_TRANSFER_LAGRANGE_7:
-        priv->transfer_cont    = interp_transfer_line_cont_8;
-        priv->transfer_generic = interp_transfer_line_generic_8;
+        priv->transfer1d_cont    = interp1d_transfer_line_cont_8;
+        priv->transfer1d_generic = interp1d_transfer_line_generic_8;
+        priv->transfer2d_cont    = interp2d_transfer_line_cont_8;
+        priv->transfer2d_generic = interp2d_transfer_line_generic_8;
         priv->stencil = 8;
         break;
     default:
         return -EINVAL;
     }
 
-    for (int dim = 0; dim < 2; dim++) {
+    priv->idx  = calloc(ctx->nb_dims, sizeof(*priv->idx));
+    priv->fact = calloc(ctx->nb_dims, sizeof(*priv->fact));
+    if (!priv->idx || !priv->fact)
+        return -ENOMEM;
+
+    for (int dim = 0; dim < ctx->nb_dims; dim++) {
         priv->idx[dim] = calloc(ctx->dst.size[dim], sizeof(*priv->idx[dim]));
         if (!priv->idx[dim])
             return -ENOMEM;
@@ -210,13 +231,17 @@ static void transfer_lagrange_free(GridTransferContext *ctx)
 {
     GridTransferLagrange *priv = ctx->priv;
 
-    for (int dim = 0; dim < 2; dim++) {
-        free(priv->idx[dim]);
-        free(priv->fact[dim]);
+    for (int dim = 0; dim < ctx->nb_dims; dim++) {
+        if (priv->idx)
+            free(priv->idx[dim]);
+        if (priv->fact)
+            free(priv->fact[dim]);
     }
+    free(priv->idx);
+    free(priv->fact);
 }
 
-static int transfer_interp_task(void *arg, unsigned int job_idx, unsigned int thread_idx)
+static int transfer_interp2d_task(void *arg, unsigned int job_idx, unsigned int thread_idx)
 {
     GridTransferLagrange *priv = arg;
     const NDArray         *src = priv->src;
@@ -226,13 +251,13 @@ static int transfer_interp_task(void *arg, unsigned int job_idx, unsigned int th
     const double       *fact_y = priv->fact[0] + priv->stencil * job_idx;
 
     if (dst->stride[1] == 1 && src->stride[1] == 1) {
-        priv->transfer_cont(dst->data + job_idx * dst->stride[0], dst->shape[1],
-                            src->data + idx_y * src->stride[0], src->stride[0],
-                            priv->idx[1], priv->fact[1], fact_y);
+        priv->transfer2d_cont(dst->data + job_idx * dst->stride[0], dst->shape[1],
+                              src->data + idx_y * src->stride[0], src->stride[0],
+                              priv->idx[1], priv->fact[1], fact_y);
     } else {
-        priv->transfer_generic(dst->data + job_idx * dst->stride[0], dst->shape[1],
-                               src->data + idx_y * src->stride[0], src->stride[0],
-                               priv->idx[1], priv->fact[1], fact_y, dst->stride[1], src->stride[1]);
+        priv->transfer2d_generic(dst->data + job_idx * dst->stride[0], dst->shape[1],
+                                 src->data + idx_y * src->stride[0], src->stride[0],
+                                 priv->idx[1], priv->fact[1], fact_y, dst->stride[1], src->stride[1]);
     }
 
     return 0;
@@ -246,7 +271,15 @@ static int transfer_lagrange_transfer(GridTransferContext *ctx,
     priv->src = src;
     priv->dst = dst;
 
-    tp_execute(ctx->tp, ctx->dst.size[0], transfer_interp_task, priv);
+    if (ctx->nb_dims == 1) {
+        if (dst->stride[0] == 1 && src->stride[0] == 1) {
+            priv->transfer1d_cont(dst->data, dst->shape[0], src->data, priv->idx[0], priv->fact[0]);
+        } else {
+            priv->transfer1d_generic(dst->data, dst->shape[0], src->data, priv->idx[0], priv->fact[0],
+                                     dst->stride[0], src->stride[0]);
+        }
+    } else
+        tp_execute(ctx->tp, ctx->dst.size[0], transfer_interp2d_task, priv);
 
     priv->src = NULL;
     priv->dst = NULL;
@@ -325,7 +358,10 @@ static int transfer_fw_init(GridTransferContext *ctx)
 {
     GridTransferFW *fw = ctx->priv;
 
-    for (int i = 0; i < 2; i++) {
+    if (ctx->nb_dims != 2)
+        return -ENOSYS;
+
+    for (int i = 0; i < ctx->nb_dims; i++) {
         /* the destination must have twice the stepsize of the source */
         if (fabs(ctx->dst.step[i] / ctx->src.step[i] - 2.0) > 1e-14)
             return -EINVAL;
@@ -376,15 +412,18 @@ int mg2di_gt_transfer(GridTransferContext *ctx,
 {
     const GridTransfer *t = transfer_ops[ctx->op];
 
-    if (dst->dims != 2 || src->dims != 2 ||
-        dst->shape[0] != ctx->dst.size[0] || dst->shape[1] != ctx->dst.size[1] ||
-        src->shape[0] != ctx->src.size[0] || src->shape[1] != ctx->src.size[1])
+    if (dst->dims != ctx->nb_dims || src->dims != ctx->nb_dims)
         return -EINVAL;
 
+    for (int dir = 0; dir < ctx->nb_dims; dir++) {
+        if (dst->shape[dir] != ctx->dst.size[dir] || src->shape[dir] != ctx->src.size[dir])
+            return -EINVAL;
+    }
+
     return t->transfer(ctx, dst, src);
 }
 
-GridTransferContext *mg2di_gt_alloc(enum GridTransferOperator op)
+GridTransferContext *mg2di_gt_alloc(unsigned int nb_dims, enum GridTransferOperator op)
 {
     GridTransferContext *ctx;
     const GridTransfer *t;
@@ -397,16 +436,28 @@ GridTransferContext *mg2di_gt_alloc(enum GridTransferOperator op)
     if (!ctx)
         return NULL;
 
+    ctx->src.start = calloc(nb_dims, sizeof(*ctx->src.start));
+    ctx->src.size  = calloc(nb_dims, sizeof(*ctx->src.size));
+    ctx->src.step  = calloc(nb_dims, sizeof(*ctx->src.step));
+    ctx->dst.start = calloc(nb_dims, sizeof(*ctx->dst.start));
+    ctx->dst.size  = calloc(nb_dims, sizeof(*ctx->dst.size));
+    ctx->dst.step  = calloc(nb_dims, sizeof(*ctx->dst.step));
+    if (!ctx->src.start || !ctx->src.size || !ctx->src.step ||
+        !ctx->dst.start || !ctx->dst.size || !ctx->dst.step)
+        goto fail;
+    ctx->nb_dims = nb_dims;
+
     if (t->priv_data_size) {
         ctx->priv = calloc(1, t->priv_data_size);
-        if (!ctx->priv) {
-            free(ctx);
-            return NULL;
-        }
+        if (!ctx->priv)
+            goto fail;
     }
     ctx->op = op;
 
     return ctx;
+fail:
+    mg2di_gt_free(&ctx);
+    return NULL;
 }
 
 int mg2di_gt_init(GridTransferContext *ctx)
@@ -418,7 +469,7 @@ int mg2di_gt_init(GridTransferContext *ctx)
         return -EINVAL;
 
     /* check that the destination grid is inside the source one */
-    for (int i = 0; i < 2; i++) {
+    for (int i = 0; i < ctx->nb_dims; i++) {
         double src_start, src_end, dst_start, dst_end;
 
         if (ctx->src.step[i] <= 0.0 || ctx->dst.step[i] <= 0.0 ||
@@ -459,6 +510,13 @@ void mg2di_gt_free(GridTransferContext **pctx)
 
     free(ctx->priv);
 
+    free(ctx->src.start);
+    free(ctx->src.size);
+    free(ctx->src.step);
+    free(ctx->dst.start);
+    free(ctx->dst.size);
+    free(ctx->dst.step);
+
     free(ctx);
     *pctx = NULL;
 }