From 60d1383db4d9f646ce5504edc9b2c437836696db Mon Sep 17 00:00:00 2001 From: Anton Khirnov Date: Sat, 29 Jun 2019 10:23:40 +0200 Subject: transfer: implement and use 1D interpolation Stop abusing "2D of y size 1" for this. --- transfer.c | 156 ++++++++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 107 insertions(+), 49 deletions(-) (limited to 'transfer.c') diff --git a/transfer.c b/transfer.c index 9c5e125..ee635b9 100644 --- a/transfer.c +++ b/transfer.c @@ -41,30 +41,35 @@ typedef struct GridTransfer { typedef struct GridTransferLagrange { unsigned int stencil; - void (*transfer_cont) (double *dst, ptrdiff_t dst_len, - const double *src, ptrdiff_t src_stride, - const ptrdiff_t *idx_x, const double *fact_x, const double *fact_y); - void (*transfer_generic)(double *dst, ptrdiff_t dst_len, - const double *src, ptrdiff_t src_stride, - const ptrdiff_t *idx_x, const double *fact_x, const double *fact_y, - ptrdiff_t dst_stride0, ptrdiff_t src_stride0); + void (*transfer2d_cont) (double *dst, ptrdiff_t dst_len, + const double *src, ptrdiff_t src_stride, + const ptrdiff_t *idx_x, const double *fact_x, const double *fact_y); + void (*transfer2d_generic)(double *dst, ptrdiff_t dst_len, + const double *src, ptrdiff_t src_stride, + const ptrdiff_t *idx_x, const double *fact_x, const double *fact_y, + ptrdiff_t dst_stride0, ptrdiff_t src_stride0); + void (*transfer1d_cont) (double *dst, ptrdiff_t dst_len, + const double *src, const ptrdiff_t *idx_x, const double *fact_x); + void (*transfer1d_generic)(double *dst, ptrdiff_t dst_len, + const double *src, const ptrdiff_t *idx_x, const double *fact_x, + ptrdiff_t dst_stride0, ptrdiff_t src_stride0); const NDArray *src; NDArray *dst; - ptrdiff_t *idx[2]; - double *fact[2]; + ptrdiff_t **idx; + double **fact; } GridTransferLagrange; #if HAVE_EXTERNAL_ASM -void mg2di_transfer_interp_line_cont_4_fma3(double *dst, ptrdiff_t dst_len, - const double *src, ptrdiff_t src_stride, - const ptrdiff_t *idx_x, - const double *fact_x, const double *fact_y); -void mg2di_transfer_interp_line_cont_6_fma3(double *dst, ptrdiff_t dst_len, - const double *src, ptrdiff_t src_stride, - const ptrdiff_t *idx_x, - const double *fact_x, const double *fact_y); +void mg2di_transfer_interp2d_line_cont_4_fma3(double *dst, ptrdiff_t dst_len, + const double *src, ptrdiff_t src_stride, + const ptrdiff_t *idx_x, + const double *fact_x, const double *fact_y); +void mg2di_transfer_interp2d_line_cont_6_fma3(double *dst, ptrdiff_t dst_len, + const double *src, ptrdiff_t src_stride, + const ptrdiff_t *idx_x, + const double *fact_x, const double *fact_y); #endif #define STENCIL 2 @@ -153,43 +158,59 @@ static int transfer_lagrange_init(GridTransferContext *ctx) GridTransferLagrange *priv = ctx->priv; int ret; + if (ctx->nb_dims > 2) + return -ENOSYS; + switch (ctx->op) { case GRID_TRANSFER_LAGRANGE_1: priv->stencil = 2; - priv->transfer_cont = interp_transfer_line_cont_2; - priv->transfer_generic = interp_transfer_line_generic_2; + priv->transfer1d_cont = interp1d_transfer_line_cont_2; + priv->transfer1d_generic = interp1d_transfer_line_generic_2; + priv->transfer2d_cont = interp2d_transfer_line_cont_2; + priv->transfer2d_generic = interp2d_transfer_line_generic_2; break; case GRID_TRANSFER_LAGRANGE_3: - priv->transfer_cont = interp_transfer_line_cont_4; - priv->transfer_generic = interp_transfer_line_generic_4; + priv->transfer1d_cont = interp1d_transfer_line_cont_4; + priv->transfer1d_generic = interp1d_transfer_line_generic_4; + priv->transfer2d_cont = interp2d_transfer_line_cont_4; + priv->transfer2d_generic = interp2d_transfer_line_generic_4; priv->stencil = 4; #if HAVE_EXTERNAL_ASM if (ctx->cpuflags & MG2DI_CPU_FLAG_FMA3) { - priv->transfer_cont = mg2di_transfer_interp_line_cont_4_fma3; + priv->transfer2d_cont = mg2di_transfer_interp2d_line_cont_4_fma3; } #endif break; case GRID_TRANSFER_LAGRANGE_5: - priv->transfer_cont = interp_transfer_line_cont_6; - priv->transfer_generic = interp_transfer_line_generic_6; + priv->transfer1d_cont = interp1d_transfer_line_cont_6; + priv->transfer1d_generic = interp1d_transfer_line_generic_6; + priv->transfer2d_cont = interp2d_transfer_line_cont_6; + priv->transfer2d_generic = interp2d_transfer_line_generic_6; priv->stencil = 6; #if HAVE_EXTERNAL_ASM if (ctx->cpuflags & MG2DI_CPU_FLAG_FMA3) { - priv->transfer_cont = mg2di_transfer_interp_line_cont_6_fma3; + priv->transfer2d_cont = mg2di_transfer_interp2d_line_cont_6_fma3; } #endif break; case GRID_TRANSFER_LAGRANGE_7: - priv->transfer_cont = interp_transfer_line_cont_8; - priv->transfer_generic = interp_transfer_line_generic_8; + priv->transfer1d_cont = interp1d_transfer_line_cont_8; + priv->transfer1d_generic = interp1d_transfer_line_generic_8; + priv->transfer2d_cont = interp2d_transfer_line_cont_8; + priv->transfer2d_generic = interp2d_transfer_line_generic_8; priv->stencil = 8; break; default: return -EINVAL; } - for (int dim = 0; dim < 2; dim++) { + priv->idx = calloc(ctx->nb_dims, sizeof(*priv->idx)); + priv->fact = calloc(ctx->nb_dims, sizeof(*priv->fact)); + if (!priv->idx || !priv->fact) + return -ENOMEM; + + for (int dim = 0; dim < ctx->nb_dims; dim++) { priv->idx[dim] = calloc(ctx->dst.size[dim], sizeof(*priv->idx[dim])); if (!priv->idx[dim]) return -ENOMEM; @@ -210,13 +231,17 @@ static void transfer_lagrange_free(GridTransferContext *ctx) { GridTransferLagrange *priv = ctx->priv; - for (int dim = 0; dim < 2; dim++) { - free(priv->idx[dim]); - free(priv->fact[dim]); + for (int dim = 0; dim < ctx->nb_dims; dim++) { + if (priv->idx) + free(priv->idx[dim]); + if (priv->fact) + free(priv->fact[dim]); } + free(priv->idx); + free(priv->fact); } -static int transfer_interp_task(void *arg, unsigned int job_idx, unsigned int thread_idx) +static int transfer_interp2d_task(void *arg, unsigned int job_idx, unsigned int thread_idx) { GridTransferLagrange *priv = arg; const NDArray *src = priv->src; @@ -226,13 +251,13 @@ static int transfer_interp_task(void *arg, unsigned int job_idx, unsigned int th const double *fact_y = priv->fact[0] + priv->stencil * job_idx; if (dst->stride[1] == 1 && src->stride[1] == 1) { - priv->transfer_cont(dst->data + job_idx * dst->stride[0], dst->shape[1], - src->data + idx_y * src->stride[0], src->stride[0], - priv->idx[1], priv->fact[1], fact_y); + priv->transfer2d_cont(dst->data + job_idx * dst->stride[0], dst->shape[1], + src->data + idx_y * src->stride[0], src->stride[0], + priv->idx[1], priv->fact[1], fact_y); } else { - priv->transfer_generic(dst->data + job_idx * dst->stride[0], dst->shape[1], - src->data + idx_y * src->stride[0], src->stride[0], - priv->idx[1], priv->fact[1], fact_y, dst->stride[1], src->stride[1]); + priv->transfer2d_generic(dst->data + job_idx * dst->stride[0], dst->shape[1], + src->data + idx_y * src->stride[0], src->stride[0], + priv->idx[1], priv->fact[1], fact_y, dst->stride[1], src->stride[1]); } return 0; @@ -246,7 +271,15 @@ static int transfer_lagrange_transfer(GridTransferContext *ctx, priv->src = src; priv->dst = dst; - tp_execute(ctx->tp, ctx->dst.size[0], transfer_interp_task, priv); + if (ctx->nb_dims == 1) { + if (dst->stride[0] == 1 && src->stride[0] == 1) { + priv->transfer1d_cont(dst->data, dst->shape[0], src->data, priv->idx[0], priv->fact[0]); + } else { + priv->transfer1d_generic(dst->data, dst->shape[0], src->data, priv->idx[0], priv->fact[0], + dst->stride[0], src->stride[0]); + } + } else + tp_execute(ctx->tp, ctx->dst.size[0], transfer_interp2d_task, priv); priv->src = NULL; priv->dst = NULL; @@ -325,7 +358,10 @@ static int transfer_fw_init(GridTransferContext *ctx) { GridTransferFW *fw = ctx->priv; - for (int i = 0; i < 2; i++) { + if (ctx->nb_dims != 2) + return -ENOSYS; + + for (int i = 0; i < ctx->nb_dims; i++) { /* the destination must have twice the stepsize of the source */ if (fabs(ctx->dst.step[i] / ctx->src.step[i] - 2.0) > 1e-14) return -EINVAL; @@ -376,15 +412,18 @@ int mg2di_gt_transfer(GridTransferContext *ctx, { const GridTransfer *t = transfer_ops[ctx->op]; - if (dst->dims != 2 || src->dims != 2 || - dst->shape[0] != ctx->dst.size[0] || dst->shape[1] != ctx->dst.size[1] || - src->shape[0] != ctx->src.size[0] || src->shape[1] != ctx->src.size[1]) + if (dst->dims != ctx->nb_dims || src->dims != ctx->nb_dims) return -EINVAL; + for (int dir = 0; dir < ctx->nb_dims; dir++) { + if (dst->shape[dir] != ctx->dst.size[dir] || src->shape[dir] != ctx->src.size[dir]) + return -EINVAL; + } + return t->transfer(ctx, dst, src); } -GridTransferContext *mg2di_gt_alloc(enum GridTransferOperator op) +GridTransferContext *mg2di_gt_alloc(unsigned int nb_dims, enum GridTransferOperator op) { GridTransferContext *ctx; const GridTransfer *t; @@ -397,16 +436,28 @@ GridTransferContext *mg2di_gt_alloc(enum GridTransferOperator op) if (!ctx) return NULL; + ctx->src.start = calloc(nb_dims, sizeof(*ctx->src.start)); + ctx->src.size = calloc(nb_dims, sizeof(*ctx->src.size)); + ctx->src.step = calloc(nb_dims, sizeof(*ctx->src.step)); + ctx->dst.start = calloc(nb_dims, sizeof(*ctx->dst.start)); + ctx->dst.size = calloc(nb_dims, sizeof(*ctx->dst.size)); + ctx->dst.step = calloc(nb_dims, sizeof(*ctx->dst.step)); + if (!ctx->src.start || !ctx->src.size || !ctx->src.step || + !ctx->dst.start || !ctx->dst.size || !ctx->dst.step) + goto fail; + ctx->nb_dims = nb_dims; + if (t->priv_data_size) { ctx->priv = calloc(1, t->priv_data_size); - if (!ctx->priv) { - free(ctx); - return NULL; - } + if (!ctx->priv) + goto fail; } ctx->op = op; return ctx; +fail: + mg2di_gt_free(&ctx); + return NULL; } int mg2di_gt_init(GridTransferContext *ctx) @@ -418,7 +469,7 @@ int mg2di_gt_init(GridTransferContext *ctx) return -EINVAL; /* check that the destination grid is inside the source one */ - for (int i = 0; i < 2; i++) { + for (int i = 0; i < ctx->nb_dims; i++) { double src_start, src_end, dst_start, dst_end; if (ctx->src.step[i] <= 0.0 || ctx->dst.step[i] <= 0.0 || @@ -459,6 +510,13 @@ void mg2di_gt_free(GridTransferContext **pctx) free(ctx->priv); + free(ctx->src.start); + free(ctx->src.size); + free(ctx->src.step); + free(ctx->dst.start); + free(ctx->dst.size); + free(ctx->dst.step); + free(ctx); *pctx = NULL; } -- cgit v1.2.3