11#ifndef __INCLUDE_DAO_DMA_H__
12#define __INCLUDE_DAO_DMA_H__
16#include <rte_dmadev.h>
20#include <dao_config.h>
25#define DAO_DMA_MAX_POINTER 15u
28#define DAO_DMA_MAX_META_POINTER 48
31#define DAO_DMA_MAX_POINTER_THR_DFLT 8u
34#define DAO_DMA_MAX_VCHAN_PER_LCORE 64
37#define DAO_DMA_MAX_INFLIGHT_MDATA 4096
236static __rte_always_inline
int
257static __rte_always_inline
bool
260 uint16_t head = vchan->
head;
261 uint16_t tail = vchan->
tail;
263 if (vchan->
src_i && (tail == op_idx))
266 return head <= tail ? (op_idx < head || op_idx >= tail) : (op_idx < head && op_idx >= tail);
279static __rte_always_inline
bool
284 uint64_t flags = (uint64_t)vchan->
auto_free << 3;
287 if (likely((src_avail >= (
int)avail || !vchan->
src_i) &&
288 (dst_avail >= (
int)avail || !vchan->
dst_i)))
292 vchan->
dst_i, flags);
293 if (unlikely(rc < 0)) {
318static __rte_always_inline uint16_t
334static __rte_always_inline uint16_t
350static __rte_always_inline
struct rte_dma_sge *
364static __rte_always_inline
struct rte_dma_sge *
387static __rte_always_inline
void
391 uint16_t src_i = vchan->
src_i;
392 uint16_t dst_i = vchan->
dst_i;
394 vchan->
dst[dst_i].addr = dst;
395 vchan->
dst[dst_i].length = dst_len;
396 vchan->
src[src_i].addr = src;
397 vchan->
src[src_i].length = src_len;
399 vchan->
src_i = src_i + 1;
400 vchan->
dst_i = dst_i + 1;
416static __rte_always_inline
void
419 uint16_t dst_i = vchan->
dst_i;
421 vchan->
dst[dst_i].addr = dst;
422 vchan->
dst[dst_i].length = dst_len;
424 vchan->
dst_i = dst_i + 1;
440static __rte_always_inline
void
443 uint16_t src_i = vchan->
src_i;
445 vchan->
src[src_i].addr = src;
446 vchan->
src[src_i].length = src_len;
448 vchan->
src_i = src_i + 1;
463static __rte_always_inline uint16_t
466 struct rte_dma_sge *src, *dst;
467 uint16_t src_i = vchan->
src_i;
468 uint16_t dst_i = vchan->
dst_i;
469 int src_avail = vchan->
flush_thr - src_i;
472 src = vchan->
src + src_i;
473 dst = vchan->
dst + dst_i;
474 if (src_avail >= 4) {
475 vst1q_u64((uint64_t *)&src[0], vsrc[0]);
476 vst1q_u64((uint64_t *)&src[1], vsrc[1]);
477 vst1q_u64((uint64_t *)&src[2], vsrc[2]);
478 vst1q_u64((uint64_t *)&src[3], vsrc[3]);
480 vst1q_u64((uint64_t *)&dst[0], vdst[0]);
481 vst1q_u64((uint64_t *)&dst[1], vdst[1]);
482 vst1q_u64((uint64_t *)&dst[2], vdst[2]);
483 vst1q_u64((uint64_t *)&dst[3], vdst[3]);
485 vchan->
src_i = src_i + 4;
486 vchan->
dst_i = dst_i + 4;
491 while (i < 4 && src_avail > 0) {
492 vst1q_u64((uint64_t *)src, vsrc[i]);
493 vst1q_u64((uint64_t *)dst, vdst[i]);
499 vchan->
src_i = src_i + i;
500 vchan->
dst_i = dst_i + i;
505 src_i = vchan->
src_i;
506 dst_i = vchan->
dst_i;
507 src = vchan->
src + src_i;
508 dst = vchan->
dst + dst_i;
511 while (i < 4 && src_avail > 0) {
512 vst1q_u64((uint64_t *)src, vsrc[i]);
513 vst1q_u64((uint64_t *)dst, vdst[i]);
530static __rte_always_inline
void
537 cmpl = rte_dma_completed(vchan->
devid, vchan->
vchan, 128, NULL, &has_err);
538 if (unlikely(has_err)) {
554static __rte_always_inline
void
557 uint32_t cmpl, i, j, idx = 0;
561 cmpl = rte_dma_completed(vchan->
devid, vchan->
vchan, 128, NULL, &has_err);
562 if (unlikely(has_err)) {
566 for (i = vchan->
head; i < vchan->
head + cmpl; i++) {
568 for (j = 0; j < vchan->
mdata[idx].
cnt; j++) {
597static __rte_always_inline
void
599 uint16_t *pend_ptr, uint16_t pend_val, uint16_t tail)
620static __rte_always_inline
void
623 uint32_t cmpl, i, j, idx = 0;
627 cmpl = rte_dma_completed(vchan->
devid, vchan->
vchan, 128, NULL, &has_err);
628 if (unlikely(has_err)) {
632 for (i = vchan->
head; i < vchan->
head + cmpl; i++) {
634 for (j = 0; j < vchan->
mdata[idx].
cnt; j++) {
658static __rte_always_inline
void
static __rte_always_inline void dao_dma_enq_src_x1(struct dao_dma_vchan_state *vchan, rte_iova_t src, uint32_t src_len)
static __rte_always_inline int dao_dma_has_stats_feature(void)
static __rte_always_inline struct rte_dma_sge * dao_dma_sge_dst(struct dao_dma_vchan_state *vchan)
static __rte_always_inline void dao_dma_check_meta_compl(struct dao_dma_vchan_state *vchan, const int mem_order)
void dao_dma_compl_wait(uint16_t vchan)
int dao_dma_lcore_mem2dev_autofree_set(int16_t dma_devid, uint16_t vchan, bool enable)
#define DAO_DMA_MAX_INFLIGHT_MDATA
int16_t dao_dma_ctrl_mem2dev(void)
static __rte_always_inline void dao_dma_check_compl(struct dao_dma_vchan_state *vchan)
int dao_dma_lcore_mem2dev_set(int16_t dma_devid, uint16_t nb_vchans, uint16_t flush_thr)
static __rte_always_inline void dao_dma_enq_x1(struct dao_dma_vchan_state *vchan, rte_iova_t src, uint32_t src_len, rte_iova_t dst, uint32_t dst_len)
int dao_dma_stats_get(uint16_t lcore_id, struct dao_dma_stats *stats)
#define DAO_DMA_MAX_VCHAN_PER_LCORE
#define DAO_DMA_MAX_META_POINTER
static __rte_always_inline uint16_t dao_dma_src_avail(struct dao_dma_vchan_state *vchan)
int dao_dma_flush_submit(void)
RTE_DECLARE_PER_LCORE(struct dao_dma_vchan_info *, dao_dma_vchan_info)
static __rte_always_inline void dao_dma_enq_dst_x1(struct dao_dma_vchan_state *vchan, rte_iova_t dst, uint32_t dst_len)
static __rte_always_inline void dao_dma_check_meta_compl_v2(struct dao_dma_vchan_state *vchan, const int mem_order)
static __rte_always_inline uint16_t dao_dma_dst_avail(struct dao_dma_vchan_state *vchan)
static __rte_always_inline bool dao_dma_flush(struct dao_dma_vchan_state *vchan, const uint8_t avail)
static __rte_always_inline struct rte_dma_sge * dao_dma_sge_src(struct dao_dma_vchan_state *vchan)
static __rte_always_inline void dao_dma_update_cmpl_meta(struct dao_dma_vchan_state *vchan, uint16_t *ptr, uint16_t val, uint16_t *pend_ptr, uint16_t pend_val, uint16_t tail)
static __rte_always_inline void dao_dma_update_cmpl_meta_v2(struct dao_dma_vchan_state *vchan, uint16_t *ptr, uint16_t val, uint16_t tail)
static __rte_always_inline bool dao_dma_op_status(struct dao_dma_vchan_state *vchan, uint16_t op_idx)
int16_t dao_dma_ctrl_dev2mem(void)
static __rte_always_inline uint16_t dao_dma_enq_x4(struct dao_dma_vchan_state *vchan, uint64x2_t *vsrc, uint64x2_t *vdst)
int dao_dma_ctrl_dev_set(int16_t dev2mem_id, int16_t mem2dev_id)
int dao_dma_lcore_dev2mem_set(int16_t dma_devid, uint16_t nb_vchans, uint16_t flush_thr)
#define DAO_DMA_MAX_POINTER
uint16_t val[DAO_DMA_MAX_META_POINTER]
uint16_t * pend_ptr[DAO_DMA_MAX_META_POINTER]
uint16_t * ptr[DAO_DMA_MAX_META_POINTER]
uint16_t pend_val[DAO_DMA_MAX_META_POINTER]
struct dao_dma_vchan_stats dev2mem[DAO_DMA_MAX_VCHAN_PER_LCORE]
struct dao_dma_vchan_stats mem2dev[DAO_DMA_MAX_VCHAN_PER_LCORE]
struct dao_dma_vchan_state mem2dev[DAO_DMA_MAX_VCHAN_PER_LCORE]
struct dao_dma_vchan_state dev2mem[DAO_DMA_MAX_VCHAN_PER_LCORE]
struct rte_dma_sge dst[DAO_DMA_MAX_POINTER]
struct rte_dma_sge src[DAO_DMA_MAX_POINTER]
struct dao_dma_cmpl_mdata mdata[DAO_DMA_MAX_INFLIGHT_MDATA]