Data Accelerator Offload
Loading...
Searching...
No Matches
dao_dma.h
Go to the documentation of this file.
1/* SPDX-License-Identifier: Marvell-MIT
2 * Copyright (c) 2023 Marvell.
3 */
4
11#ifndef __INCLUDE_DAO_DMA_H__
12#define __INCLUDE_DAO_DMA_H__
13
14#include <rte_eal.h>
15
16#include <rte_dmadev.h>
17#include <rte_lcore.h>
18#include <rte_vect.h>
19
20#include <dao_config.h>
21
22#include "dao_log.h"
23
25#define DAO_DMA_MAX_POINTER 15u
26
28#define DAO_DMA_MAX_META_POINTER 48
29
31#define DAO_DMA_MAX_POINTER_THR_DFLT 8u
32
34#define DAO_DMA_MAX_VCHAN_PER_LCORE 64
35
37#define DAO_DMA_MAX_INFLIGHT_MDATA 4096
38
52
56 uint16_t tail;
58 uint16_t head;
60 int16_t devid;
62 uint8_t vchan;
63 uint8_t rsvd;
65 uint16_t src_i;
67 uint16_t dst_i;
69 uint8_t flush_thr;
71 uint8_t auto_free : 1;
72 uint8_t rsvd2 : 7;
74 uint16_t pend_ops;
76 struct rte_dma_sge src[DAO_DMA_MAX_POINTER];
78 struct rte_dma_sge dst[DAO_DMA_MAX_POINTER];
80 uint64_t ptrs;
82 uint64_t ops;
84 uint64_t dbells;
86 uint64_t dma_enq_errs;
92
104
108 uint64_t ptrs;
110 uint64_t ops;
112 uint64_t dbells;
114 uint64_t enq_errs;
115};
116
128
131
139
150int dao_dma_stats_get(uint16_t lcore_id, struct dao_dma_stats *stats);
151
164int dao_dma_lcore_dev2mem_set(int16_t dma_devid, uint16_t nb_vchans, uint16_t flush_thr);
165
178int dao_dma_lcore_mem2dev_set(int16_t dma_devid, uint16_t nb_vchans, uint16_t flush_thr);
179
192int dao_dma_lcore_mem2dev_autofree_set(int16_t dma_devid, uint16_t vchan, bool enable);
193
204int dao_dma_ctrl_dev_set(int16_t dev2mem_id, int16_t mem2dev_id);
205
213
221
228void dao_dma_compl_wait(uint16_t vchan);
229
236static __rte_always_inline int
238{
239#if DAO_DMA_STATS
240 return 1;
241#else
242 return 0;
243#endif
244}
245
257static __rte_always_inline bool
258dao_dma_op_status(struct dao_dma_vchan_state *vchan, uint16_t op_idx)
259{
260 uint16_t head = vchan->head;
261 uint16_t tail = vchan->tail;
262
263 if (vchan->src_i && (tail == op_idx))
264 return false;
265
266 return head <= tail ? (op_idx < head || op_idx >= tail) : (op_idx < head && op_idx >= tail);
267}
268
279static __rte_always_inline bool
280dao_dma_flush(struct dao_dma_vchan_state *vchan, const uint8_t avail)
281{
282 int src_avail = vchan->flush_thr - vchan->src_i;
283 int dst_avail = vchan->flush_thr - vchan->dst_i;
284 uint64_t flags = (uint64_t)vchan->auto_free << 3;
285 int rc;
286
287 if (likely((src_avail >= (int)avail || !vchan->src_i) &&
288 (dst_avail >= (int)avail || !vchan->dst_i)))
289 goto exit;
290
291 rc = rte_dma_copy_sg(vchan->devid, vchan->vchan, vchan->src, vchan->dst, vchan->src_i,
292 vchan->dst_i, flags);
293 if (unlikely(rc < 0)) {
295 vchan->dma_enq_errs++;
296 return false;
297 }
298 vchan->tail++;
299 vchan->pend_ops++;
301 vchan->ptrs += vchan->src_i;
302 vchan->ops++;
303 }
304 vchan->src_i = 0;
305 vchan->dst_i = 0;
306exit:
307 return true;
308}
309
318static __rte_always_inline uint16_t
320{
321 int src_avail = vchan->flush_thr - vchan->src_i;
322
323 return src_avail;
324}
325
334static __rte_always_inline uint16_t
336{
337 int dst_avail = vchan->flush_thr - vchan->dst_i;
338
339 return dst_avail;
340}
341
350static __rte_always_inline struct rte_dma_sge *
352{
353 return &vchan->src[vchan->src_i];
354}
355
364static __rte_always_inline struct rte_dma_sge *
366{
367 return &vchan->dst[vchan->dst_i];
368}
369
387static __rte_always_inline void
388dao_dma_enq_x1(struct dao_dma_vchan_state *vchan, rte_iova_t src, uint32_t src_len, rte_iova_t dst,
389 uint32_t dst_len)
390{
391 uint16_t src_i = vchan->src_i;
392 uint16_t dst_i = vchan->dst_i;
393
394 vchan->dst[dst_i].addr = dst;
395 vchan->dst[dst_i].length = dst_len;
396 vchan->src[src_i].addr = src;
397 vchan->src[src_i].length = src_len;
398
399 vchan->src_i = src_i + 1;
400 vchan->dst_i = dst_i + 1;
401}
402
416static __rte_always_inline void
417dao_dma_enq_dst_x1(struct dao_dma_vchan_state *vchan, rte_iova_t dst, uint32_t dst_len)
418{
419 uint16_t dst_i = vchan->dst_i;
420
421 vchan->dst[dst_i].addr = dst;
422 vchan->dst[dst_i].length = dst_len;
423
424 vchan->dst_i = dst_i + 1;
425}
426
440static __rte_always_inline void
441dao_dma_enq_src_x1(struct dao_dma_vchan_state *vchan, rte_iova_t src, uint32_t src_len)
442{
443 uint16_t src_i = vchan->src_i;
444
445 vchan->src[src_i].addr = src;
446 vchan->src[src_i].length = src_len;
447
448 vchan->src_i = src_i + 1;
449}
450
463static __rte_always_inline uint16_t
464dao_dma_enq_x4(struct dao_dma_vchan_state *vchan, uint64x2_t *vsrc, uint64x2_t *vdst)
465{
466 struct rte_dma_sge *src, *dst;
467 uint16_t src_i = vchan->src_i;
468 uint16_t dst_i = vchan->dst_i;
469 int src_avail = vchan->flush_thr - src_i;
470 int i;
471
472 src = vchan->src + src_i;
473 dst = vchan->dst + dst_i;
474 if (src_avail >= 4) {
475 vst1q_u64((uint64_t *)&src[0], vsrc[0]);
476 vst1q_u64((uint64_t *)&src[1], vsrc[1]);
477 vst1q_u64((uint64_t *)&src[2], vsrc[2]);
478 vst1q_u64((uint64_t *)&src[3], vsrc[3]);
479
480 vst1q_u64((uint64_t *)&dst[0], vdst[0]);
481 vst1q_u64((uint64_t *)&dst[1], vdst[1]);
482 vst1q_u64((uint64_t *)&dst[2], vdst[2]);
483 vst1q_u64((uint64_t *)&dst[3], vdst[3]);
484
485 vchan->src_i = src_i + 4;
486 vchan->dst_i = dst_i + 4;
487 return 4;
488 }
489
490 i = 0;
491 while (i < 4 && src_avail > 0) {
492 vst1q_u64((uint64_t *)src, vsrc[i]);
493 vst1q_u64((uint64_t *)dst, vdst[i]);
494 src++;
495 dst++;
496 i++;
497 src_avail--;
498 };
499 vchan->src_i = src_i + i;
500 vchan->dst_i = dst_i + i;
501
502 /* Flush enqueued pointers */
503 dao_dma_flush(vchan, 4);
504
505 src_i = vchan->src_i;
506 dst_i = vchan->dst_i;
507 src = vchan->src + src_i;
508 dst = vchan->dst + dst_i;
509 src_avail = vchan->flush_thr - src_i;
510
511 while (i < 4 && src_avail > 0) {
512 vst1q_u64((uint64_t *)src, vsrc[i]);
513 vst1q_u64((uint64_t *)dst, vdst[i]);
514 i++;
515 src++;
516 dst++;
517 src_avail--;
518 vchan->src_i++;
519 vchan->dst_i++;
520 };
521 return i;
522}
523
530static __rte_always_inline void
532{
533 uint16_t cmpl;
534 bool has_err = 0;
535
536 /* Fetch all DMA completed status */
537 cmpl = rte_dma_completed(vchan->devid, vchan->vchan, 128, NULL, &has_err);
538 if (unlikely(has_err)) {
539 vchan->dma_compl_errs++;
540 cmpl += 1;
541 }
542 vchan->head += cmpl;
543}
544
554static __rte_always_inline void
555dao_dma_check_meta_compl(struct dao_dma_vchan_state *vchan, const int mem_order)
556{
557 uint32_t cmpl, i, j, idx = 0;
558 bool has_err = 0;
559
560 /* Fetch all DMA completed status */
561 cmpl = rte_dma_completed(vchan->devid, vchan->vchan, 128, NULL, &has_err);
562 if (unlikely(has_err)) {
563 vchan->dma_compl_errs++;
564 cmpl += 1;
565 }
566 for (i = vchan->head; i < vchan->head + cmpl; i++) {
568 for (j = 0; j < vchan->mdata[idx].cnt; j++) {
569 if (mem_order)
570 __atomic_store_n(vchan->mdata[idx].ptr[j], vchan->mdata[idx].val[j],
571 __ATOMIC_RELEASE);
572 else
573 *vchan->mdata[idx].ptr[j] = vchan->mdata[idx].val[j];
574 *vchan->mdata[idx].pend_ptr[j] -= vchan->mdata[idx].pend_val[j];
575 }
576 vchan->mdata[idx].cnt = 0;
577 }
578 vchan->head += cmpl;
579}
580
597static __rte_always_inline void
598dao_dma_update_cmpl_meta(struct dao_dma_vchan_state *vchan, uint16_t *ptr, uint16_t val,
599 uint16_t *pend_ptr, uint16_t pend_val, uint16_t tail)
600{
601 uint16_t idx = tail % DAO_DMA_MAX_INFLIGHT_MDATA;
602 uint16_t j = vchan->mdata[idx].cnt;
603
604 vchan->mdata[idx].ptr[j] = ptr;
605 vchan->mdata[idx].val[j] = val;
606 vchan->mdata[idx].pend_ptr[j] = pend_ptr;
607 vchan->mdata[idx].pend_val[j] = pend_val;
608 vchan->mdata[idx].cnt = j + 1;
609}
610
620static __rte_always_inline void
621dao_dma_check_meta_compl_v2(struct dao_dma_vchan_state *vchan, const int mem_order)
622{
623 uint32_t cmpl, i, j, idx = 0;
624 bool has_err = 0;
625
626 /* Fetch all DMA completed status */
627 cmpl = rte_dma_completed(vchan->devid, vchan->vchan, 128, NULL, &has_err);
628 if (unlikely(has_err)) {
629 vchan->dma_compl_errs++;
630 cmpl += 1;
631 }
632 for (i = vchan->head; i < vchan->head + cmpl; i++) {
634 for (j = 0; j < vchan->mdata[idx].cnt; j++) {
635 if (mem_order)
636 __atomic_store_n(vchan->mdata[idx].ptr[j], vchan->mdata[idx].val[j],
637 __ATOMIC_RELAXED);
638 else
639 *vchan->mdata[idx].ptr[j] = vchan->mdata[idx].val[j];
640 }
641 vchan->mdata[idx].cnt = 0;
642 }
643 vchan->head += cmpl;
644}
645
658static __rte_always_inline void
659dao_dma_update_cmpl_meta_v2(struct dao_dma_vchan_state *vchan, uint16_t *ptr, uint16_t val,
660 uint16_t tail)
661{
662 uint16_t idx = tail % DAO_DMA_MAX_INFLIGHT_MDATA;
663 uint16_t j = vchan->mdata[idx].cnt;
664
665 vchan->mdata[idx].ptr[j] = ptr;
666 vchan->mdata[idx].val[j] = val;
667 vchan->mdata[idx].cnt = j + 1;
668}
669
670#endif /* __INCLUDE_DAO_DMA_H__ */
static __rte_always_inline void dao_dma_enq_src_x1(struct dao_dma_vchan_state *vchan, rte_iova_t src, uint32_t src_len)
Definition dao_dma.h:441
static __rte_always_inline int dao_dma_has_stats_feature(void)
Definition dao_dma.h:237
static __rte_always_inline struct rte_dma_sge * dao_dma_sge_dst(struct dao_dma_vchan_state *vchan)
Definition dao_dma.h:365
static __rte_always_inline void dao_dma_check_meta_compl(struct dao_dma_vchan_state *vchan, const int mem_order)
Definition dao_dma.h:555
void dao_dma_compl_wait(uint16_t vchan)
int dao_dma_lcore_mem2dev_autofree_set(int16_t dma_devid, uint16_t vchan, bool enable)
#define DAO_DMA_MAX_INFLIGHT_MDATA
Definition dao_dma.h:37
int16_t dao_dma_ctrl_mem2dev(void)
static __rte_always_inline void dao_dma_check_compl(struct dao_dma_vchan_state *vchan)
Definition dao_dma.h:531
int dao_dma_lcore_mem2dev_set(int16_t dma_devid, uint16_t nb_vchans, uint16_t flush_thr)
static __rte_always_inline void dao_dma_enq_x1(struct dao_dma_vchan_state *vchan, rte_iova_t src, uint32_t src_len, rte_iova_t dst, uint32_t dst_len)
Definition dao_dma.h:388
int dao_dma_stats_get(uint16_t lcore_id, struct dao_dma_stats *stats)
#define DAO_DMA_MAX_VCHAN_PER_LCORE
Definition dao_dma.h:34
#define DAO_DMA_MAX_META_POINTER
Definition dao_dma.h:28
static __rte_always_inline uint16_t dao_dma_src_avail(struct dao_dma_vchan_state *vchan)
Definition dao_dma.h:319
int dao_dma_flush_submit(void)
RTE_DECLARE_PER_LCORE(struct dao_dma_vchan_info *, dao_dma_vchan_info)
static __rte_always_inline void dao_dma_enq_dst_x1(struct dao_dma_vchan_state *vchan, rte_iova_t dst, uint32_t dst_len)
Definition dao_dma.h:417
static __rte_always_inline void dao_dma_check_meta_compl_v2(struct dao_dma_vchan_state *vchan, const int mem_order)
Definition dao_dma.h:621
static __rte_always_inline uint16_t dao_dma_dst_avail(struct dao_dma_vchan_state *vchan)
Definition dao_dma.h:335
static __rte_always_inline bool dao_dma_flush(struct dao_dma_vchan_state *vchan, const uint8_t avail)
Definition dao_dma.h:280
static __rte_always_inline struct rte_dma_sge * dao_dma_sge_src(struct dao_dma_vchan_state *vchan)
Definition dao_dma.h:351
static __rte_always_inline void dao_dma_update_cmpl_meta(struct dao_dma_vchan_state *vchan, uint16_t *ptr, uint16_t val, uint16_t *pend_ptr, uint16_t pend_val, uint16_t tail)
Definition dao_dma.h:598
static __rte_always_inline void dao_dma_update_cmpl_meta_v2(struct dao_dma_vchan_state *vchan, uint16_t *ptr, uint16_t val, uint16_t tail)
Definition dao_dma.h:659
static __rte_always_inline bool dao_dma_op_status(struct dao_dma_vchan_state *vchan, uint16_t op_idx)
Definition dao_dma.h:258
int16_t dao_dma_ctrl_dev2mem(void)
static __rte_always_inline uint16_t dao_dma_enq_x4(struct dao_dma_vchan_state *vchan, uint64x2_t *vsrc, uint64x2_t *vdst)
Definition dao_dma.h:464
int dao_dma_ctrl_dev_set(int16_t dev2mem_id, int16_t mem2dev_id)
int dao_dma_lcore_dev2mem_set(int16_t dma_devid, uint16_t nb_vchans, uint16_t flush_thr)
#define DAO_DMA_MAX_POINTER
Definition dao_dma.h:25
uint16_t val[DAO_DMA_MAX_META_POINTER]
Definition dao_dma.h:46
uint16_t * pend_ptr[DAO_DMA_MAX_META_POINTER]
Definition dao_dma.h:42
uint16_t * ptr[DAO_DMA_MAX_META_POINTER]
Definition dao_dma.h:48
uint16_t pend_val[DAO_DMA_MAX_META_POINTER]
Definition dao_dma.h:44
uint16_t nb_mem2dev
Definition dao_dma.h:122
struct dao_dma_vchan_stats dev2mem[DAO_DMA_MAX_VCHAN_PER_LCORE]
Definition dao_dma.h:124
struct dao_dma_vchan_stats mem2dev[DAO_DMA_MAX_VCHAN_PER_LCORE]
Definition dao_dma.h:126
uint16_t nb_dev2mem
Definition dao_dma.h:120
struct dao_dma_vchan_state mem2dev[DAO_DMA_MAX_VCHAN_PER_LCORE]
Definition dao_dma.h:102
uint16_t nb_mem2dev
Definition dao_dma.h:98
struct dao_dma_vchan_state dev2mem[DAO_DMA_MAX_VCHAN_PER_LCORE]
Definition dao_dma.h:100
uint16_t nb_dev2mem
Definition dao_dma.h:96
uint64_t dma_enq_errs
Definition dao_dma.h:86
uint8_t flush_thr
Definition dao_dma.h:69
struct rte_dma_sge dst[DAO_DMA_MAX_POINTER]
Definition dao_dma.h:78
uint8_t auto_free
Definition dao_dma.h:71
uint16_t pend_ops
Definition dao_dma.h:74
uint64_t dma_compl_errs
Definition dao_dma.h:88
uint64_t dbells
Definition dao_dma.h:84
struct rte_dma_sge src[DAO_DMA_MAX_POINTER]
Definition dao_dma.h:76
struct dao_dma_cmpl_mdata mdata[DAO_DMA_MAX_INFLIGHT_MDATA]
Definition dao_dma.h:90
uint64_t enq_errs
Definition dao_dma.h:114