dao_dma.h

Go to the documentation of this file.

/* SPDX-License-Identifier: Marvell-MIT
 * Copyright (c) 2023 Marvell.
 */
 
#ifndef __INCLUDE_DAO_DMA_H__
#define __INCLUDE_DAO_DMA_H__
 
#include <rte_eal.h>
 
#include <rte_dmadev.h>
#include <rte_lcore.h>
#include <rte_mempool.h>
#include <rte_prefetch.h>
#include <rte_vect.h>
 
#include <dao_config.h>
 
#include "dao_log.h"
 
#define DAO_DMA_MAX_POINTER 15u
 
#define DAO_DMA_MAX_META_POINTER 48
 
#define DAO_DMA_MAX_VCHAN_PER_LCORE 64
 
#define DAO_DMA_MAX_INFLIGHT_MDATA 4096
 
#define DAO_DMA_DOORBELL_THRESHOLD 1024
 
struct dao_dma_cmpl_mdata {
    uint16_t *pend_ptr[DAO_DMA_MAX_META_POINTER];
    uint16_t pend_val[DAO_DMA_MAX_META_POINTER];
    union {
        uint16_t val[DAO_DMA_MAX_META_POINTER];
        uint32_t val32[DAO_DMA_MAX_META_POINTER];
    };
    uint16_t *ptr[DAO_DMA_MAX_META_POINTER];
    uint16_t cnt;
};
 
struct dao_dma_vchan_state {
    uint16_t tail;
    uint16_t head;
    int16_t devid;
    uint8_t vchan;
    uint8_t rsvd;
    uint16_t src_i;
    uint16_t dst_i;
    uint8_t flush_thr;
    uint8_t auto_free : 1;
    uint8_t rsvd2 : 7;
    uint16_t pend_ops;
    struct rte_dma_sge src[DAO_DMA_MAX_POINTER];
    struct rte_dma_sge dst[DAO_DMA_MAX_POINTER];
    uint64_t ptrs;
    uint64_t ops;
    uint64_t dbells;
    uint64_t dma_enq_errs;
    uint64_t dma_compl_errs;
    void *ops_mem;
    struct rte_dma_op **dma_ops;
    uint16_t ops_head;
    uint16_t ops_tail;
    uint16_t ops_mask;
    struct dao_dma_cmpl_mdata mdata[DAO_DMA_MAX_INFLIGHT_MDATA];
} __rte_cache_aligned;
 
struct dao_dma_vchan_info {
    uint16_t nb_dev2mem;
    uint16_t nb_mem2dev;
    struct dao_dma_vchan_state dev2mem[DAO_DMA_MAX_VCHAN_PER_LCORE];
    struct dao_dma_vchan_state mem2dev[DAO_DMA_MAX_VCHAN_PER_LCORE];
} __rte_cache_aligned;
 
struct dao_dma_vchan_stats {
    uint64_t ptrs;
    uint64_t ops;
    uint64_t dbells;
    uint64_t enq_errs;
};
 
struct dao_dma_stats {
    uint16_t nb_dev2mem;
    uint16_t nb_mem2dev;
    struct dao_dma_vchan_stats dev2mem[DAO_DMA_MAX_VCHAN_PER_LCORE];
    struct dao_dma_vchan_stats mem2dev[DAO_DMA_MAX_VCHAN_PER_LCORE];
};
 
RTE_DECLARE_PER_LCORE(struct dao_dma_vchan_info *, dao_dma_vchan_info);
 
int dao_dma_flush_submit(void);
 
int dao_dma_flush_submit_v2(void);
 
int dao_dma_flush_submit_ops(void);
 
int dao_dma_stats_get(uint16_t lcore_id, struct dao_dma_stats *stats);
 
int dao_dma_lcore_dev2mem_set(int16_t dma_devid, uint16_t nb_vchans, uint16_t flush_thr);
 
int dao_dma_lcore_dev2mem_set_ops(int16_t dma_devid, uint16_t nb_vchans, uint16_t flush_thr,
                  uint16_t nb_ops);
 
int dao_dma_lcore_mem2dev_set(int16_t dma_devid, uint16_t nb_vchans, uint16_t flush_thr);
 
int dao_dma_lcore_mem2dev_set_ops(int16_t dma_devid, uint16_t nb_vchans, uint16_t flush_thr,
                  uint16_t nb_ops);
 
int dao_dma_lcore_mem2dev_autofree_set(int16_t dma_devid, uint16_t vchan, bool enable);
 
int dao_dma_ctrl_dev_set(int16_t dev2mem_id, int16_t mem2dev_id);
 
int16_t dao_dma_ctrl_dev2mem(void);
 
int16_t dao_dma_ctrl_mem2dev(void);
 
void dao_dma_compl_wait_inflight(uint16_t vchan);
 
static __rte_always_inline int
dao_dma_has_stats_feature(void)
{
#if DAO_DMA_STATS
    return 1;
#else
    return 0;
#endif
}
 
static __rte_always_inline uint16_t
dao_dma_burst_capacity(struct dao_dma_vchan_state *vchan)
{
    return rte_dma_burst_capacity(vchan->devid, vchan->vchan);
}
 
static __rte_always_inline bool
dao_dma_desc_avail_get(struct dao_dma_vchan_state *vchan, uint32_t nb_src, uint32_t nb_dst)
{
    int src_avail = vchan->flush_thr - vchan->src_i;
    int dst_avail = vchan->flush_thr - vchan->dst_i;
    uint16_t inflight = vchan->tail - vchan->head;
    uint32_t n_src, n_dst, nb_desc;
    uint8_t flush_thr;
 
    if (likely((src_avail >= (int)nb_src || !vchan->src_i) &&
           (dst_avail >= (int)nb_dst || !vchan->dst_i)))
        return true;
 
    flush_thr = vchan->flush_thr;
    n_src = (nb_src + flush_thr - 1) / flush_thr;
    n_dst = (nb_dst + flush_thr - 1) / flush_thr;
    nb_desc = RTE_MAX(n_src, n_dst);
 
    /* increment by 1 to consider current enqueued pointers
     */
    nb_desc++;
    if (inflight + nb_desc > DAO_DMA_MAX_INFLIGHT_MDATA)
        return false;
 
    return dao_dma_burst_capacity(vchan) >= nb_desc;
}
 
static __rte_always_inline bool
dao_dma_op_status(struct dao_dma_vchan_state *vchan, uint16_t op_idx)
{
    uint16_t head = vchan->head;
    uint16_t tail = vchan->tail;
 
    if (vchan->src_i && (tail == op_idx))
        return false;
 
    return head <= tail ? (op_idx < head || op_idx >= tail) : (op_idx < head && op_idx >= tail);
}
 
static __rte_always_inline bool
dao_dma_flush(struct dao_dma_vchan_state *vchan, const uint8_t avail)
{
    int src_avail = vchan->flush_thr - vchan->src_i;
    int dst_avail = vchan->flush_thr - vchan->dst_i;
    uint64_t flags = (uint64_t)vchan->auto_free << 3;
    int rc;
 
    if (likely((src_avail >= (int)avail || !vchan->src_i) &&
           (dst_avail >= (int)avail || !vchan->dst_i)))
        goto exit;
 
    if (vchan->pend_ops >= DAO_DMA_DOORBELL_THRESHOLD)
        flags |= RTE_DMA_OP_FLAG_SUBMIT;
 
    rc = rte_dma_copy_sg(vchan->devid, vchan->vchan, vchan->src, vchan->dst, vchan->src_i,
                 vchan->dst_i, flags);
    if (unlikely(rc < 0)) {
        if (dao_dma_has_stats_feature())
            vchan->dma_enq_errs++;
        return false;
    }
    vchan->tail++;
    vchan->pend_ops++;
 
    if (flags & RTE_DMA_OP_FLAG_SUBMIT)
        vchan->pend_ops = 0;
 
    if (dao_dma_has_stats_feature()) {
        vchan->ptrs += vchan->src_i;
        vchan->ops++;
    }
    vchan->src_i = 0;
    vchan->dst_i = 0;
exit:
    return true;
}
 
static __rte_always_inline uint16_t
dao_dma_src_avail(struct dao_dma_vchan_state *vchan)
{
    int src_avail = vchan->flush_thr - vchan->src_i;
 
    return src_avail;
}
 
static __rte_always_inline uint16_t
dao_dma_dst_avail(struct dao_dma_vchan_state *vchan)
{
    int dst_avail = vchan->flush_thr - vchan->dst_i;
 
    return dst_avail;
}
 
static __rte_always_inline struct rte_dma_sge *
dao_dma_sge_src(struct dao_dma_vchan_state *vchan)
{
    return &vchan->src[vchan->src_i];
}
 
static __rte_always_inline struct rte_dma_sge *
dao_dma_sge_dst(struct dao_dma_vchan_state *vchan)
{
    return &vchan->dst[vchan->dst_i];
}
 
static __rte_always_inline void
dao_dma_enq_x1(struct dao_dma_vchan_state *vchan, rte_iova_t src, uint32_t src_len, rte_iova_t dst,
           uint32_t dst_len)
{
    uint16_t src_i = vchan->src_i;
    uint16_t dst_i = vchan->dst_i;
 
    vchan->dst[dst_i].addr = dst;
    vchan->dst[dst_i].length = dst_len;
    vchan->src[src_i].addr = src;
    vchan->src[src_i].length = src_len;
 
    vchan->src_i = src_i + 1;
    vchan->dst_i = dst_i + 1;
}
 
static __rte_always_inline void
dao_dma_enq_dst_x1(struct dao_dma_vchan_state *vchan, rte_iova_t dst, uint32_t dst_len)
{
    uint16_t dst_i = vchan->dst_i;
 
    vchan->dst[dst_i].addr = dst;
    vchan->dst[dst_i].length = dst_len;
 
    vchan->dst_i = dst_i + 1;
}
 
static __rte_always_inline void
dao_dma_enq_src_x1(struct dao_dma_vchan_state *vchan, rte_iova_t src, uint32_t src_len)
{
    uint16_t src_i = vchan->src_i;
 
    vchan->src[src_i].addr = src;
    vchan->src[src_i].length = src_len;
 
    vchan->src_i = src_i + 1;
}
 
static __rte_always_inline uint16_t
dao_dma_enq_x4(struct dao_dma_vchan_state *vchan, uint64x2_t *vsrc, uint64x2_t *vdst)
{
    struct rte_dma_sge *src, *dst;
    uint16_t src_i = vchan->src_i;
    uint16_t dst_i = vchan->dst_i;
    int src_avail = vchan->flush_thr - src_i;
    int i;
 
    src = vchan->src + src_i;
    dst = vchan->dst + dst_i;
    if (src_avail >= 4) {
        vst1q_u64((uint64_t *)&src[0], vsrc[0]);
        vst1q_u64((uint64_t *)&src[1], vsrc[1]);
        vst1q_u64((uint64_t *)&src[2], vsrc[2]);
        vst1q_u64((uint64_t *)&src[3], vsrc[3]);
 
        vst1q_u64((uint64_t *)&dst[0], vdst[0]);
        vst1q_u64((uint64_t *)&dst[1], vdst[1]);
        vst1q_u64((uint64_t *)&dst[2], vdst[2]);
        vst1q_u64((uint64_t *)&dst[3], vdst[3]);
 
        vchan->src_i = src_i + 4;
        vchan->dst_i = dst_i + 4;
        return 4;
    }
 
    i = 0;
    while (i < 4 && src_avail > 0) {
        vst1q_u64((uint64_t *)src, vsrc[i]);
        vst1q_u64((uint64_t *)dst, vdst[i]);
        src++;
        dst++;
        i++;
        src_avail--;
    };
    vchan->src_i = src_i + i;
    vchan->dst_i = dst_i + i;
 
    /* Flush enqueued pointers */
    dao_dma_flush(vchan, 4);
 
    src_i = vchan->src_i;
    dst_i = vchan->dst_i;
    src = vchan->src + src_i;
    dst = vchan->dst + dst_i;
    src_avail = vchan->flush_thr - src_i;
 
    while (i < 4 && src_avail > 0) {
        vst1q_u64((uint64_t *)src, vsrc[i]);
        vst1q_u64((uint64_t *)dst, vdst[i]);
        i++;
        src++;
        dst++;
        src_avail--;
        vchan->src_i++;
        vchan->dst_i++;
    };
    return i;
}
 
static __rte_always_inline void
dao_dma_check_compl(struct dao_dma_vchan_state *vchan)
{
    uint16_t cmpl;
    bool has_err = 0;
 
    /* Fetch all DMA completed status */
    cmpl = rte_dma_completed(vchan->devid, vchan->vchan, 128, NULL, &has_err);
    if (unlikely(has_err)) {
        vchan->dma_compl_errs++;
        cmpl += 1;
    }
    vchan->head += cmpl;
}
 
static __rte_always_inline uint16_t
dao_dma_ops_avail(struct dao_dma_vchan_state *vchan)
{
    return vchan->ops_head - vchan->ops_tail;
}
 
static __rte_always_inline struct rte_dma_op **
dao_dma_ops_get(struct dao_dma_vchan_state *vchan, uint16_t n)
{
    uint16_t tail = vchan->ops_tail;
 
    vchan->ops_tail = tail + n;
    return &vchan->dma_ops[tail & vchan->ops_mask];
}
 
static __rte_always_inline void
dao_dma_ops_put(struct dao_dma_vchan_state *vchan, uint16_t n)
{
    vchan->ops_head += n;
}
 
static __rte_always_inline void
dao_dma_ops_release(struct dao_dma_vchan_state *vchan, uint16_t n)
{
    vchan->ops_tail -= n;
}
 
static __rte_always_inline void
dao_dma_op_set_cmpl(struct rte_dma_op *op, uint16_t *ptr, uint16_t val, uint16_t *pend_ptr,
            uint16_t pend_val)
{
    op->user_meta = (uint64_t)(uintptr_t)ptr;
    op->event_meta = (uint64_t)(uintptr_t)pend_ptr;
    op->rsvd = ((uint32_t)val << 16) | pend_val;
}
 
static __rte_always_inline void
dao_dma_check_meta_compl(struct dao_dma_vchan_state *vchan, const int mem_order)
{
    uint32_t cmpl, i, j, idx = 0;
    bool has_err = 0;
 
    /* Fetch all DMA completed status */
    cmpl = rte_dma_completed(vchan->devid, vchan->vchan, 128, NULL, &has_err);
    if (unlikely(has_err)) {
        vchan->dma_compl_errs++;
        cmpl += 1;
    }
    for (i = vchan->head; i < vchan->head + cmpl; i++) {
        idx = i % DAO_DMA_MAX_INFLIGHT_MDATA;
        for (j = 0; j < vchan->mdata[idx].cnt; j++) {
            if (mem_order)
                __atomic_store_n(vchan->mdata[idx].ptr[j], vchan->mdata[idx].val[j],
                         __ATOMIC_RELEASE);
            else
                *vchan->mdata[idx].ptr[j] = vchan->mdata[idx].val[j];
            *vchan->mdata[idx].pend_ptr[j] -= vchan->mdata[idx].pend_val[j];
        }
        vchan->mdata[idx].cnt = 0;
    }
    vchan->head += cmpl;
}
 
static __rte_always_inline void
dao_dma_check_meta_compl_ops(struct dao_dma_vchan_state *vchan, const int mem_order)
{
#define DEQ_SZ 128
    uint32_t cmpl, i;
    struct rte_dma_op *deq_ops[DEQ_SZ];
 
    /* Skip costly dequeue call when no ops are inflight */
    if (vchan->head == vchan->tail)
        return;
 
    cmpl = rte_dma_dequeue_ops(vchan->devid, vchan->vchan, deq_ops, DEQ_SZ);
    if (!cmpl)
        return;
 
    for (i = 0; i < cmpl; i++) {
        struct rte_dma_op *op = deq_ops[i];
 
        if (unlikely(op->status != RTE_DMA_STATUS_SUCCESSFUL))
            vchan->dma_compl_errs++;
 
        if (op->rsvd) {
            uint16_t *ptr = (uint16_t *)(uintptr_t)op->user_meta;
            uint16_t *pend_ptr = (uint16_t *)(uintptr_t)op->event_meta;
            uint16_t val = op->rsvd >> 16;
            uint16_t pend_val = op->rsvd & 0xFFFF;
 
            if (mem_order)
                __atomic_store_n(ptr, val, __ATOMIC_RELEASE);
            else
                *ptr = val;
            *pend_ptr -= pend_val;
            op->rsvd = 0;
        }
    }
 
    /* Return ops to ring buffer */
    dao_dma_ops_put(vchan, cmpl);
    vchan->head += cmpl;
}
 
static __rte_always_inline void
dao_dma_update_cmpl_meta(struct dao_dma_vchan_state *vchan, uint16_t *ptr, uint16_t val,
             uint16_t *pend_ptr, uint16_t pend_val, uint16_t tail)
{
    uint16_t idx = tail % DAO_DMA_MAX_INFLIGHT_MDATA;
    uint16_t j = vchan->mdata[idx].cnt;
 
    vchan->mdata[idx].ptr[j] = ptr;
    vchan->mdata[idx].val[j] = val;
    vchan->mdata[idx].pend_ptr[j] = pend_ptr;
    vchan->mdata[idx].pend_val[j] = pend_val;
    vchan->mdata[idx].cnt = j + 1;
}
 
static __rte_always_inline void
dao_dma_check_meta_compl_v2(struct dao_dma_vchan_state *vchan, const int mem_order)
{
    uint32_t cmpl, i, j, idx = 0;
    bool has_err = 0;
    uint32_t *ptr;
 
    /* Fetch all DMA completed status */
    cmpl = rte_dma_completed(vchan->devid, vchan->vchan, 128, NULL, &has_err);
    if (unlikely(has_err)) {
        vchan->dma_compl_errs++;
        cmpl += 1;
    }
    for (i = vchan->head; i < vchan->head + cmpl; i++) {
        idx = i % DAO_DMA_MAX_INFLIGHT_MDATA;
        for (j = 0; j < vchan->mdata[idx].cnt; j++) {
            ptr = (uint32_t *)vchan->mdata[idx].ptr[j];
            if (mem_order)
                __atomic_store_n(ptr, vchan->mdata[idx].val32[j], __ATOMIC_RELEASE);
            else
                *ptr = vchan->mdata[idx].val32[j];
        }
        vchan->mdata[idx].cnt = 0;
    }
    vchan->head += cmpl;
}
 
static __rte_always_inline void
dao_dma_update_cmpl_meta_v2(struct dao_dma_vchan_state *vchan, uint32_t *ptr, uint32_t val,
                uint16_t tail)
{
    uint16_t idx = tail % DAO_DMA_MAX_INFLIGHT_MDATA;
    uint16_t j = vchan->mdata[idx].cnt;
 
    vchan->mdata[idx].ptr[j] = (uint16_t *)ptr;
    vchan->mdata[idx].val32[j] = val;
    vchan->mdata[idx].cnt = j + 1;
}
 
#endif /* __INCLUDE_DAO_DMA_H__ */