dao_dma.h File Reference

#include <rte_eal.h>
#include <rte_dmadev.h>
#include <rte_lcore.h>
#include <rte_mempool.h>
#include <rte_prefetch.h>
#include <rte_vect.h>
#include <dao_config.h>
#include "dao_log.h"

Go to the source code of this file.

Data Structures
struct	dao_dma_cmpl_mdata
struct	dao_dma_vchan_state
struct	dao_dma_vchan_info
struct	dao_dma_vchan_stats
struct	dao_dma_stats

Macros
#define	DAO_DMA_MAX_POINTER   15u
#define	DAO_DMA_MAX_META_POINTER   48
#define	DAO_DMA_MAX_VCHAN_PER_LCORE   64
#define	DAO_DMA_MAX_INFLIGHT_MDATA   4096
#define	DAO_DMA_DOORBELL_THRESHOLD   1024

Functions
	RTE_DECLARE_PER_LCORE (struct dao_dma_vchan_info *, dao_dma_vchan_info)
int	dao_dma_flush_submit (void)
int	dao_dma_flush_submit_v2 (void)
int	dao_dma_flush_submit_ops (void)
int	dao_dma_stats_get (uint16_t lcore_id, struct dao_dma_stats *stats)
int	dao_dma_lcore_dev2mem_set (int16_t dma_devid, uint16_t nb_vchans, uint16_t flush_thr)
int	dao_dma_lcore_dev2mem_set_ops (int16_t dma_devid, uint16_t nb_vchans, uint16_t flush_thr, uint16_t nb_ops)
int	dao_dma_lcore_mem2dev_set (int16_t dma_devid, uint16_t nb_vchans, uint16_t flush_thr)
int	dao_dma_lcore_mem2dev_set_ops (int16_t dma_devid, uint16_t nb_vchans, uint16_t flush_thr, uint16_t nb_ops)
int	dao_dma_lcore_mem2dev_autofree_set (int16_t dma_devid, uint16_t vchan, bool enable)
int	dao_dma_ctrl_dev_set (int16_t dev2mem_id, int16_t mem2dev_id)
int16_t	dao_dma_ctrl_dev2mem (void)
int16_t	dao_dma_ctrl_mem2dev (void)
void	dao_dma_compl_wait_inflight (uint16_t vchan)
static __rte_always_inline int	dao_dma_has_stats_feature (void)
static __rte_always_inline uint16_t	dao_dma_burst_capacity (struct dao_dma_vchan_state *vchan)
static __rte_always_inline bool	dao_dma_desc_avail_get (struct dao_dma_vchan_state *vchan, uint32_t nb_src, uint32_t nb_dst)
static __rte_always_inline bool	dao_dma_op_status (struct dao_dma_vchan_state *vchan, uint16_t op_idx)
static __rte_always_inline bool	dao_dma_flush (struct dao_dma_vchan_state *vchan, const uint8_t avail)
static __rte_always_inline uint16_t	dao_dma_src_avail (struct dao_dma_vchan_state *vchan)
static __rte_always_inline uint16_t	dao_dma_dst_avail (struct dao_dma_vchan_state *vchan)
static __rte_always_inline struct rte_dma_sge *	dao_dma_sge_src (struct dao_dma_vchan_state *vchan)
static __rte_always_inline struct rte_dma_sge *	dao_dma_sge_dst (struct dao_dma_vchan_state *vchan)
static __rte_always_inline void	dao_dma_enq_x1 (struct dao_dma_vchan_state *vchan, rte_iova_t src, uint32_t src_len, rte_iova_t dst, uint32_t dst_len)
static __rte_always_inline void	dao_dma_enq_dst_x1 (struct dao_dma_vchan_state *vchan, rte_iova_t dst, uint32_t dst_len)
static __rte_always_inline void	dao_dma_enq_src_x1 (struct dao_dma_vchan_state *vchan, rte_iova_t src, uint32_t src_len)
static __rte_always_inline uint16_t	dao_dma_enq_x4 (struct dao_dma_vchan_state *vchan, uint64x2_t *vsrc, uint64x2_t *vdst)
static __rte_always_inline void	dao_dma_check_compl (struct dao_dma_vchan_state *vchan)
static __rte_always_inline uint16_t	dao_dma_ops_avail (struct dao_dma_vchan_state *vchan)
static __rte_always_inline struct rte_dma_op **	dao_dma_ops_get (struct dao_dma_vchan_state *vchan, uint16_t n)
static __rte_always_inline void	dao_dma_ops_put (struct dao_dma_vchan_state *vchan, uint16_t n)
static __rte_always_inline void	dao_dma_ops_release (struct dao_dma_vchan_state *vchan, uint16_t n)
static __rte_always_inline void	dao_dma_op_set_cmpl (struct rte_dma_op *op, uint16_t *ptr, uint16_t val, uint16_t *pend_ptr, uint16_t pend_val)
static __rte_always_inline void	dao_dma_check_meta_compl (struct dao_dma_vchan_state *vchan, const int mem_order)
static __rte_always_inline void	dao_dma_check_meta_compl_ops (struct dao_dma_vchan_state *vchan, const int mem_order)
static __rte_always_inline void	dao_dma_update_cmpl_meta (struct dao_dma_vchan_state *vchan, uint16_t *ptr, uint16_t val, uint16_t *pend_ptr, uint16_t pend_val, uint16_t tail)
static __rte_always_inline void	dao_dma_check_meta_compl_v2 (struct dao_dma_vchan_state *vchan, const int mem_order)
static __rte_always_inline void	dao_dma_update_cmpl_meta_v2 (struct dao_dma_vchan_state *vchan, uint16_t *ptr, uint16_t val, uint16_t tail)

Detailed Description

DAO DMA helper

Definition in file dao_dma.h.

Macro Definition Documentation

DAO_DMA_MAX_POINTER

#define DAO_DMA_MAX_POINTER   15u

DMA MAX pointer

Definition at line 27 of file dao_dma.h.

DAO_DMA_MAX_META_POINTER

#define DAO_DMA_MAX_META_POINTER   48

DMA MAX META pointer MAX DMA POINTER

Definition at line 30 of file dao_dma.h.

DAO_DMA_MAX_VCHAN_PER_LCORE

#define DAO_DMA_MAX_VCHAN_PER_LCORE   64

DMA Max VCHAN per lcore

Definition at line 33 of file dao_dma.h.

DAO_DMA_MAX_INFLIGHT_MDATA

#define DAO_DMA_MAX_INFLIGHT_MDATA   4096

DMA inflight event meta data

Definition at line 36 of file dao_dma.h.

DAO_DMA_DOORBELL_THRESHOLD

#define DAO_DMA_DOORBELL_THRESHOLD   1024

DMA doorbell threshold

Definition at line 39 of file dao_dma.h.

Function Documentation

RTE_DECLARE_PER_LCORE()

RTE_DECLARE_PER_LCORE	(	struct dao_dma_vchan_info *	,
		dao_dma_vchan_info
	)

DMA per lcore vchan info

dao_dma_flush_submit()

int dao_dma_flush_submit

(

void

)

Flush DMA requests and submit ops

Returns

Zero on success.

dao_dma_flush_submit_v2()

int dao_dma_flush_submit_v2

(

void

)

Flush DMA requests and submit ops version 2

Returns

Zero on success.

dao_dma_flush_submit_ops()

int dao_dma_flush_submit_ops

(

void

)

Flush DMA requests and submit ops

Returns

Zero on success.

dao_dma_stats_get()

int dao_dma_stats_get	(	uint16_t	lcore_id,
		struct dao_dma_stats *	stats
	)

Get DMA stats from DAO library

Parameters

lcore_id	Lcore to get stats from.
stats	Address to store stats.

Returns

Zero on success.

dao_dma_lcore_dev2mem_set()

int dao_dma_lcore_dev2mem_set	(	int16_t	dma_devid,
		uint16_t	nb_vchans,
		uint16_t	flush_thr
	)

Assign dev2mem dma device to an lcore.

Parameters

dma_devid	DMA device id to assign to lcore.
nb_vchans	Number of vchans available in DMA device
flush_thr	Flush threshold.

Returns

Zero on success.

dao_dma_lcore_dev2mem_set_ops()

int dao_dma_lcore_dev2mem_set_ops	(	int16_t	dma_devid,
		uint16_t	nb_vchans,
		uint16_t	flush_thr,
		uint16_t	nb_ops
	)

Assign dev2mem dma device to an lcore with ops support.

Parameters

dma_devid	DMA device id to assign to lcore.
nb_vchans	Number of vchans available in DMA device
flush_thr	Flush threshold.
nb_ops	Number of DMA operations per vchan.

Returns

Zero on success.

dao_dma_lcore_mem2dev_set()

int dao_dma_lcore_mem2dev_set	(	int16_t	dma_devid,
		uint16_t	nb_vchans,
		uint16_t	flush_thr
	)

Assign mem2dev dma device to an lcore.

Parameters

dma_devid	DMA device id to assign to lcore.
nb_vchans	Number of vchans available in DMA device
flush_thr	Flush threshold.

Returns

Zero on success.

dao_dma_lcore_mem2dev_set_ops()

int dao_dma_lcore_mem2dev_set_ops	(	int16_t	dma_devid,
		uint16_t	nb_vchans,
		uint16_t	flush_thr,
		uint16_t	nb_ops
	)

Assign mem2dev dma device to an lcore.

Parameters

dma_devid	DMA device id to assign to lcore.
nb_vchans	Number of vchans available in DMA device
flush_thr	Flush threshold.
nb_ops	Number of DMA operations per vchan.

Returns

Zero on success.

dao_dma_lcore_mem2dev_autofree_set()

int dao_dma_lcore_mem2dev_autofree_set	(	int16_t	dma_devid,
		uint16_t	vchan,
		bool	enable
	)

Enable or Disable auto free on a mem2dev dma device's vchan of a given lcore.

Parameters

dma_devid	DMA device id to assigned to the lcore.
vchan	mem2dev vchan of the dma device.
enable	Flag

Returns

Zero on success.

dao_dma_ctrl_dev_set()

int dao_dma_ctrl_dev_set	(	int16_t	dev2mem_id,
		int16_t	mem2dev_id
	)

Assign a global DMA device id for control path.

Parameters

dev2mem_id	dev2mem dma device id.
mem2dev_id	mem2dev dma device id.

Returns

Zero on success.

dao_dma_ctrl_dev2mem()

int16_t dao_dma_ctrl_dev2mem

(

void

)

Get global DMA dev2mem device id.

Returns

dma device id.

dao_dma_ctrl_mem2dev()

int16_t dao_dma_ctrl_mem2dev

(

void

)

Get global DMA mem2dev device id.

Returns

dma device id.

dao_dma_compl_wait_inflight()

void dao_dma_compl_wait_inflight

(

uint16_t

vchan

)

Check and wait for all DMA requests that are enqueued to current tail index

Parameters

vchan

Vchan ID

dao_dma_has_stats_feature()

static __rte_always_inline int dao_dma_has_stats_feature ( void  )

static

Tests DMA stats support

Returns

1 if DMA stats is supported, 0 otherwise.

Definition at line 299 of file dao_dma.h.

dao_dma_burst_capacity()

static __rte_always_inline uint16_t dao_dma_burst_capacity ( struct dao_dma_vchan_state *  vchan )

static

Query DMA descriptor capacity for a vchan.

Parameters

vchan

Vchan state pointer

Returns

Number of DMA instructions that can be submitted.

Definition at line 317 of file dao_dma.h.

dao_dma_desc_avail_get()

static __rte_always_inline bool dao_dma_desc_avail_get ( struct dao_dma_vchan_state *  vchan, uint32_t  nb_src, uint32_t  nb_dst  )

static

Check if the DMA vchan has enough descriptor capacity for a multi segment enqueue

Parameters

vchan	Vchan state pointer
nb_src	Total source segments to enqueue.
nb_dst	Total destination segments to enqueue.

Returns

True if space is available, false otherwise.

Definition at line 335 of file dao_dma.h.

dao_dma_op_status()

static __rte_always_inline bool dao_dma_op_status ( struct dao_dma_vchan_state *  vchan, uint16_t  op_idx  )

static

Get DMA operation status

Parameters

vchan	Vchan associated with DMA operation.
op_idx	DMA operation index

Returns

True - DMA op complete. False - DMA op not complete.

Definition at line 373 of file dao_dma.h.

dao_dma_flush()

static __rte_always_inline bool dao_dma_flush ( struct dao_dma_vchan_state *  vchan, const uint8_t  avail  )

static

Flush DMA operation

Parameters

vchan	Vchan state pointer
avail	Avail needed post flush

Returns

True on success.

Definition at line 395 of file dao_dma.h.

dao_dma_src_avail()

static __rte_always_inline uint16_t dao_dma_src_avail ( struct dao_dma_vchan_state *  vchan )

static

Get available space in DMA src vchan state

Parameters

vchan

Vchan state pointer

Returns

Returns space in number of src pointers.

Definition at line 441 of file dao_dma.h.

dao_dma_dst_avail()

static __rte_always_inline uint16_t dao_dma_dst_avail ( struct dao_dma_vchan_state *  vchan )

static

Get available space in DMA dst vchan state

Parameters

vchan

Vchan state pointer

Returns

Returns space in number of dst pointers.

Definition at line 457 of file dao_dma.h.

dao_dma_sge_src()

static __rte_always_inline struct rte_dma_sge * dao_dma_sge_src ( struct dao_dma_vchan_state *  vchan )

static

Get DMA src pointer base

Parameters

vchan

Vchan state pointer

Returns

Returns pointer base

Definition at line 473 of file dao_dma.h.

dao_dma_sge_dst()

static __rte_always_inline struct rte_dma_sge * dao_dma_sge_dst ( struct dao_dma_vchan_state *  vchan )

static

Get DMA dst pointer base

Parameters

vchan

Vchan state pointer

Returns

Returns pointer base

Definition at line 487 of file dao_dma.h.

dao_dma_enq_x1()

static __rte_always_inline void dao_dma_enq_x1 ( struct dao_dma_vchan_state *  vchan, rte_iova_t  src, uint32_t  src_len, rte_iova_t  dst, uint32_t  dst_len  )

static

Enqueue one DMA pointer pair.

Space is vchan state is checked by caller before calling this API

Parameters

vchan	Vchan state pointer
src	source data IOVA
src_len	source data length
dst	Destination data IOVA
dst_len	Destination data len

Definition at line 510 of file dao_dma.h.

dao_dma_enq_dst_x1()

static __rte_always_inline void dao_dma_enq_dst_x1 ( struct dao_dma_vchan_state *  vchan, rte_iova_t  dst, uint32_t  dst_len  )

static

Enqueue one DMA pointer for destination address.

Space in vchan state is checked by caller before calling this API

Parameters

vchan	Vchan state pointer
dst	Destination data IOVA
dst_len	Destination data len

Definition at line 539 of file dao_dma.h.

dao_dma_enq_src_x1()

static __rte_always_inline void dao_dma_enq_src_x1 ( struct dao_dma_vchan_state *  vchan, rte_iova_t  src, uint32_t  src_len  )

static

Enqueue one DMA pointer for source address.

Space in vchan state is checked by caller before calling this API

Parameters

vchan	Vchan state pointer
src	source data IOVA
src_len	source data length

Definition at line 563 of file dao_dma.h.

dao_dma_enq_x4()

static __rte_always_inline uint16_t dao_dma_enq_x4 ( struct dao_dma_vchan_state *  vchan, uint64x2_t *  vsrc, uint64x2_t *  vdst  )

static

Enqueue four DMA pointer pairs.

Parameters

vchan	Vchan state pointer
vsrc	source vector in format of struct rte_dma_sge.
vdst	destination vector in format of struct rte_dma_sge.

Returns

Number of pointers enqueued.

Definition at line 586 of file dao_dma.h.

dao_dma_check_compl()

static __rte_always_inline void dao_dma_check_compl ( struct dao_dma_vchan_state *  vchan )

static

Check and update DMA completions.

Parameters

vchan

Vchan state pointer

Definition at line 653 of file dao_dma.h.

dao_dma_ops_avail()

static __rte_always_inline uint16_t dao_dma_ops_avail ( struct dao_dma_vchan_state *  vchan )

static

Get available ops count from ring buffer.

Parameters

vchan

Vchan state pointer

Returns

Number of available ops in the ring.

Definition at line 676 of file dao_dma.h.

dao_dma_ops_get()

static __rte_always_inline struct rte_dma_op ** dao_dma_ops_get ( struct dao_dma_vchan_state *  vchan, uint16_t  n  )

static

Get ops pointers from ring buffer.

Parameters

vchan	Vchan state pointer
n	Number of ops requested

Returns

Pointer to contiguous ops array (2x ring avoids wrap handling).

Definition at line 692 of file dao_dma.h.

dao_dma_ops_put()

static __rte_always_inline void dao_dma_ops_put ( struct dao_dma_vchan_state *  vchan, uint16_t  n  )

static

Return ops to ring buffer (bulk) - advances head.

Parameters

vchan	Vchan state pointer
n	Number of ops to return (must match dequeue order)

Definition at line 709 of file dao_dma.h.

dao_dma_ops_release()

static __rte_always_inline void dao_dma_ops_release ( struct dao_dma_vchan_state *  vchan, uint16_t  n  )

static

Reset ops tail to release unsubmitted ops.

Parameters

vchan	Vchan state pointer
n	Number of ops to release from tail

Definition at line 723 of file dao_dma.h.

dao_dma_op_set_cmpl()

static __rte_always_inline void dao_dma_op_set_cmpl ( struct rte_dma_op *  op, uint16_t *  ptr, uint16_t  val, uint16_t *  pend_ptr, uint16_t  pend_val  )

static

Store completion metadata directly in rte_dma_op header fields.

Layout: op->user_meta = ptr (descriptor offset pointer) op->event_meta = pend_ptr (pending counter pointer) op->rsvd = val << 16 | pend_val (non-zero == valid)

Definition at line 737 of file dao_dma.h.

dao_dma_check_meta_compl()

static __rte_always_inline void dao_dma_check_meta_compl ( struct dao_dma_vchan_state *  vchan, const int  mem_order  )

static

Check and update DMA completions with metadata.

Parameters

vchan	Vchan state pointer
mem_order	Memory order to update address

Definition at line 755 of file dao_dma.h.

dao_dma_check_meta_compl_ops()

static __rte_always_inline void dao_dma_check_meta_compl_ops ( struct dao_dma_vchan_state *  vchan, const int  mem_order  )

static

Check and update DMA completions with metadata version2.

Parameters

vchan	Vchan state pointer
mem_order	Memory order to update address

Definition at line 791 of file dao_dma.h.

dao_dma_update_cmpl_meta()

static __rte_always_inline void dao_dma_update_cmpl_meta ( struct dao_dma_vchan_state *  vchan, uint16_t *  ptr, uint16_t  val, uint16_t *  pend_ptr, uint16_t  pend_val, uint16_t  tail  )

static

Update DMA op index metadata.

Parameters

vchan	Vchan state pointer
ptr	desc location to write
val	Value to store in descriptor location
pend_ptr	Pending count location to write
pend_val	Pending Value to store
tail	Meta data index to store

Definition at line 848 of file dao_dma.h.

dao_dma_check_meta_compl_v2()

static __rte_always_inline void dao_dma_check_meta_compl_v2 ( struct dao_dma_vchan_state *  vchan, const int  mem_order  )

static

Check and update DMA completions with metadata version2.

Parameters

vchan	Vchan state pointer
mem_order	Memory order to update address

Definition at line 871 of file dao_dma.h.

dao_dma_update_cmpl_meta_v2()

static __rte_always_inline void dao_dma_update_cmpl_meta_v2 ( struct dao_dma_vchan_state *  vchan, uint16_t *  ptr, uint16_t  val, uint16_t  tail  )

static

Update DMA op index metadata version2.

Parameters

vchan	Vchan state pointer
ptr	desc location to write
val	Value to store in descriptor location
tail	Meta data index to store

Definition at line 909 of file dao_dma.h.