dao_dma.h

Go to the documentation of this file.

/* SPDX-License-Identifier: Marvell-MIT
 * Copyright (c) 2023 Marvell.
 */
 
#ifndef __INCLUDE_DAO_DMA_H__
#define __INCLUDE_DAO_DMA_H__
 
#include <rte_eal.h>
 
#include <rte_dmadev.h>
#include <rte_lcore.h>
#include <rte_vect.h>
 
#include <dao_config.h>
 
#include "dao_log.h"
 
#define DAO_DMA_MAX_POINTER 15u
 
#define DAO_DMA_MAX_META_POINTER 48
 
#define DAO_DMA_MAX_POINTER_THR_DFLT 8u
 
#define DAO_DMA_MAX_VCHAN_PER_LCORE 64
 
#define DAO_DMA_MAX_INFLIGHT_MDATA 4096
 
struct dao_dma_cmpl_mdata {
    uint16_t *pend_ptr[DAO_DMA_MAX_META_POINTER];
    uint16_t pend_val[DAO_DMA_MAX_META_POINTER];
    uint16_t val[DAO_DMA_MAX_META_POINTER];
    uint16_t *ptr[DAO_DMA_MAX_META_POINTER];
    uint16_t cnt;
};
 
struct dao_dma_vchan_state {
    uint16_t tail;
    uint16_t head;
    int16_t devid;
    uint8_t vchan;
    uint8_t rsvd;
    uint16_t src_i;
    uint16_t dst_i;
    uint8_t flush_thr;
    uint8_t auto_free : 1;
    uint8_t rsvd2 : 7;
    uint16_t pend_ops;
    struct rte_dma_sge src[DAO_DMA_MAX_POINTER];
    struct rte_dma_sge dst[DAO_DMA_MAX_POINTER];
    uint64_t ptrs;
    uint64_t ops;
    uint64_t dbells;
    uint64_t dma_enq_errs;
    uint64_t dma_compl_errs;
    struct dao_dma_cmpl_mdata mdata[DAO_DMA_MAX_INFLIGHT_MDATA];
} __rte_cache_aligned;
 
struct dao_dma_vchan_info {
    uint16_t nb_dev2mem;
    uint16_t nb_mem2dev;
    struct dao_dma_vchan_state dev2mem[DAO_DMA_MAX_VCHAN_PER_LCORE];
    struct dao_dma_vchan_state mem2dev[DAO_DMA_MAX_VCHAN_PER_LCORE];
} __rte_cache_aligned;
 
struct dao_dma_vchan_stats {
    uint64_t ptrs;
    uint64_t ops;
    uint64_t dbells;
    uint64_t enq_errs;
};
 
struct dao_dma_stats {
    uint16_t nb_dev2mem;
    uint16_t nb_mem2dev;
    struct dao_dma_vchan_stats dev2mem[DAO_DMA_MAX_VCHAN_PER_LCORE];
    struct dao_dma_vchan_stats mem2dev[DAO_DMA_MAX_VCHAN_PER_LCORE];
};
 
RTE_DECLARE_PER_LCORE(struct dao_dma_vchan_info *, dao_dma_vchan_info);
 
int dao_dma_flush_submit(void);
 
int dao_dma_flush_submit_v2(void);
 
int dao_dma_stats_get(uint16_t lcore_id, struct dao_dma_stats *stats);
 
int dao_dma_lcore_dev2mem_set(int16_t dma_devid, uint16_t nb_vchans, uint16_t flush_thr);
 
int dao_dma_lcore_mem2dev_set(int16_t dma_devid, uint16_t nb_vchans, uint16_t flush_thr);
 
int dao_dma_lcore_mem2dev_autofree_set(int16_t dma_devid, uint16_t vchan, bool enable);
 
int dao_dma_ctrl_dev_set(int16_t dev2mem_id, int16_t mem2dev_id);
 
int16_t dao_dma_ctrl_dev2mem(void);
 
int16_t dao_dma_ctrl_mem2dev(void);
 
void dao_dma_compl_wait(uint16_t vchan);
 
static __rte_always_inline int
dao_dma_has_stats_feature(void)
{
#if DAO_DMA_STATS
    return 1;
#else
    return 0;
#endif
}
 
static __rte_always_inline bool
dao_dma_op_status(struct dao_dma_vchan_state *vchan, uint16_t op_idx)
{
    uint16_t head = vchan->head;
    uint16_t tail = vchan->tail;
 
    if (vchan->src_i && (tail == op_idx))
        return false;
 
    return head <= tail ? (op_idx < head || op_idx >= tail) : (op_idx < head && op_idx >= tail);
}
 
static __rte_always_inline bool
dao_dma_flush(struct dao_dma_vchan_state *vchan, const uint8_t avail)
{
    int src_avail = vchan->flush_thr - vchan->src_i;
    int dst_avail = vchan->flush_thr - vchan->dst_i;
    uint64_t flags = (uint64_t)vchan->auto_free << 3;
    int rc;
 
    if (likely((src_avail >= (int)avail || !vchan->src_i) &&
           (dst_avail >= (int)avail || !vchan->dst_i)))
        goto exit;
 
    rc = rte_dma_copy_sg(vchan->devid, vchan->vchan, vchan->src, vchan->dst, vchan->src_i,
                 vchan->dst_i, flags);
    if (unlikely(rc < 0)) {
        if (dao_dma_has_stats_feature())
            vchan->dma_enq_errs++;
        return false;
    }
    vchan->tail++;
    vchan->pend_ops++;
    if (dao_dma_has_stats_feature()) {
        vchan->ptrs += vchan->src_i;
        vchan->ops++;
    }
    vchan->src_i = 0;
    vchan->dst_i = 0;
exit:
    return true;
}
 
static __rte_always_inline uint16_t
dao_dma_src_avail(struct dao_dma_vchan_state *vchan)
{
    int src_avail = vchan->flush_thr - vchan->src_i;
 
    return src_avail;
}
 
static __rte_always_inline uint16_t
dao_dma_dst_avail(struct dao_dma_vchan_state *vchan)
{
    int dst_avail = vchan->flush_thr - vchan->dst_i;
 
    return dst_avail;
}
 
static __rte_always_inline struct rte_dma_sge *
dao_dma_sge_src(struct dao_dma_vchan_state *vchan)
{
    return &vchan->src[vchan->src_i];
}
 
static __rte_always_inline struct rte_dma_sge *
dao_dma_sge_dst(struct dao_dma_vchan_state *vchan)
{
    return &vchan->dst[vchan->dst_i];
}
 
static __rte_always_inline void
dao_dma_enq_x1(struct dao_dma_vchan_state *vchan, rte_iova_t src, uint32_t src_len, rte_iova_t dst,
           uint32_t dst_len)
{
    uint16_t src_i = vchan->src_i;
    uint16_t dst_i = vchan->dst_i;
 
    vchan->dst[dst_i].addr = dst;
    vchan->dst[dst_i].length = dst_len;
    vchan->src[src_i].addr = src;
    vchan->src[src_i].length = src_len;
 
    vchan->src_i = src_i + 1;
    vchan->dst_i = dst_i + 1;
}
 
static __rte_always_inline void
dao_dma_enq_dst_x1(struct dao_dma_vchan_state *vchan, rte_iova_t dst, uint32_t dst_len)
{
    uint16_t dst_i = vchan->dst_i;
 
    vchan->dst[dst_i].addr = dst;
    vchan->dst[dst_i].length = dst_len;
 
    vchan->dst_i = dst_i + 1;
}
 
static __rte_always_inline void
dao_dma_enq_src_x1(struct dao_dma_vchan_state *vchan, rte_iova_t src, uint32_t src_len)
{
    uint16_t src_i = vchan->src_i;
 
    vchan->src[src_i].addr = src;
    vchan->src[src_i].length = src_len;
 
    vchan->src_i = src_i + 1;
}
 
static __rte_always_inline uint16_t
dao_dma_enq_x4(struct dao_dma_vchan_state *vchan, uint64x2_t *vsrc, uint64x2_t *vdst)
{
    struct rte_dma_sge *src, *dst;
    uint16_t src_i = vchan->src_i;
    uint16_t dst_i = vchan->dst_i;
    int src_avail = vchan->flush_thr - src_i;
    int i;
 
    src = vchan->src + src_i;
    dst = vchan->dst + dst_i;
    if (src_avail >= 4) {
        vst1q_u64((uint64_t *)&src[0], vsrc[0]);
        vst1q_u64((uint64_t *)&src[1], vsrc[1]);
        vst1q_u64((uint64_t *)&src[2], vsrc[2]);
        vst1q_u64((uint64_t *)&src[3], vsrc[3]);
 
        vst1q_u64((uint64_t *)&dst[0], vdst[0]);
        vst1q_u64((uint64_t *)&dst[1], vdst[1]);
        vst1q_u64((uint64_t *)&dst[2], vdst[2]);
        vst1q_u64((uint64_t *)&dst[3], vdst[3]);
 
        vchan->src_i = src_i + 4;
        vchan->dst_i = dst_i + 4;
        return 4;
    }
 
    i = 0;
    while (i < 4 && src_avail > 0) {
        vst1q_u64((uint64_t *)src, vsrc[i]);
        vst1q_u64((uint64_t *)dst, vdst[i]);
        src++;
        dst++;
        i++;
        src_avail--;
    };
    vchan->src_i = src_i + i;
    vchan->dst_i = dst_i + i;
 
    /* Flush enqueued pointers */
    dao_dma_flush(vchan, 4);
 
    src_i = vchan->src_i;
    dst_i = vchan->dst_i;
    src = vchan->src + src_i;
    dst = vchan->dst + dst_i;
    src_avail = vchan->flush_thr - src_i;
 
    while (i < 4 && src_avail > 0) {
        vst1q_u64((uint64_t *)src, vsrc[i]);
        vst1q_u64((uint64_t *)dst, vdst[i]);
        i++;
        src++;
        dst++;
        src_avail--;
        vchan->src_i++;
        vchan->dst_i++;
    };
    return i;
}
 
static __rte_always_inline void
dao_dma_check_compl(struct dao_dma_vchan_state *vchan)
{
    uint16_t cmpl;
    bool has_err = 0;
 
    /* Fetch all DMA completed status */
    cmpl = rte_dma_completed(vchan->devid, vchan->vchan, 128, NULL, &has_err);
    if (unlikely(has_err)) {
        vchan->dma_compl_errs++;
        cmpl += 1;
    }
    vchan->head += cmpl;
}
 
static __rte_always_inline void
dao_dma_check_meta_compl(struct dao_dma_vchan_state *vchan, const int mem_order)
{
    uint32_t cmpl, i, j, idx = 0;
    bool has_err = 0;
 
    /* Fetch all DMA completed status */
    cmpl = rte_dma_completed(vchan->devid, vchan->vchan, 128, NULL, &has_err);
    if (unlikely(has_err)) {
        vchan->dma_compl_errs++;
        cmpl += 1;
    }
    for (i = vchan->head; i < vchan->head + cmpl; i++) {
        idx = i % DAO_DMA_MAX_INFLIGHT_MDATA;
        for (j = 0; j < vchan->mdata[idx].cnt; j++) {
            if (mem_order)
                __atomic_store_n(vchan->mdata[idx].ptr[j], vchan->mdata[idx].val[j],
                         __ATOMIC_RELEASE);
            else
                *vchan->mdata[idx].ptr[j] = vchan->mdata[idx].val[j];
            *vchan->mdata[idx].pend_ptr[j] -= vchan->mdata[idx].pend_val[j];
        }
        vchan->mdata[idx].cnt = 0;
    }
    vchan->head += cmpl;
}
 
static __rte_always_inline void
dao_dma_update_cmpl_meta(struct dao_dma_vchan_state *vchan, uint16_t *ptr, uint16_t val,
             uint16_t *pend_ptr, uint16_t pend_val, uint16_t tail)
{
    uint16_t idx = tail % DAO_DMA_MAX_INFLIGHT_MDATA;
    uint16_t j = vchan->mdata[idx].cnt;
 
    vchan->mdata[idx].ptr[j] = ptr;
    vchan->mdata[idx].val[j] = val;
    vchan->mdata[idx].pend_ptr[j] = pend_ptr;
    vchan->mdata[idx].pend_val[j] = pend_val;
    vchan->mdata[idx].cnt = j + 1;
}
 
static __rte_always_inline void
dao_dma_check_meta_compl_v2(struct dao_dma_vchan_state *vchan, const int mem_order)
{
    uint32_t cmpl, i, j, idx = 0;
    bool has_err = 0;
 
    /* Fetch all DMA completed status */
    cmpl = rte_dma_completed(vchan->devid, vchan->vchan, 128, NULL, &has_err);
    if (unlikely(has_err)) {
        vchan->dma_compl_errs++;
        cmpl += 1;
    }
    for (i = vchan->head; i < vchan->head + cmpl; i++) {
        idx = i % DAO_DMA_MAX_INFLIGHT_MDATA;
        for (j = 0; j < vchan->mdata[idx].cnt; j++) {
            if (mem_order)
                __atomic_store_n(vchan->mdata[idx].ptr[j], vchan->mdata[idx].val[j],
                         __ATOMIC_RELAXED);
            else
                *vchan->mdata[idx].ptr[j] = vchan->mdata[idx].val[j];
        }
        vchan->mdata[idx].cnt = 0;
    }
    vchan->head += cmpl;
}
 
static __rte_always_inline void
dao_dma_update_cmpl_meta_v2(struct dao_dma_vchan_state *vchan, uint16_t *ptr, uint16_t val,
                uint16_t tail)
{
    uint16_t idx = tail % DAO_DMA_MAX_INFLIGHT_MDATA;
    uint16_t j = vchan->mdata[idx].cnt;
 
    vchan->mdata[idx].ptr[j] = ptr;
    vchan->mdata[idx].val[j] = val;
    vchan->mdata[idx].cnt = j + 1;
}
 
#endif /* __INCLUDE_DAO_DMA_H__ */