#include <rte_eal.h>
#include <rte_dmadev.h>
#include <rte_lcore.h>
#include <rte_vect.h>
#include <dao_config.h>
#include "dao_log.h"

Data Structures
struct	dao_dma_cmpl_mdata

struct	dao_dma_vchan_state

struct	dao_dma_vchan_info

struct	dao_dma_vchan_stats

struct	dao_dma_stats

Macros
#define	DAO_DMA_MAX_POINTER 15u

#define	DAO_DMA_MAX_META_POINTER 48

#define	DAO_DMA_MAX_POINTER_THR_DFLT 8u

#define	DAO_DMA_MAX_VCHAN_PER_LCORE 64

#define	DAO_DMA_MAX_INFLIGHT_MDATA 4096

Functions
	RTE_DECLARE_PER_LCORE (struct dao_dma_vchan_info *, dao_dma_vchan_info)

int	dao_dma_flush_submit (void)

int	dao_dma_stats_get (uint16_t lcore_id, struct dao_dma_stats *stats)

int	dao_dma_lcore_dev2mem_set (int16_t dma_devid, uint16_t nb_vchans, uint16_t flush_thr)

int	dao_dma_lcore_mem2dev_set (int16_t dma_devid, uint16_t nb_vchans, uint16_t flush_thr)

int	dao_dma_lcore_mem2dev_autofree_set (int16_t dma_devid, uint16_t vchan, bool enable)

int	dao_dma_ctrl_dev_set (int16_t dev2mem_id, int16_t mem2dev_id)

int16_t	dao_dma_ctrl_dev2mem (void)

int16_t	dao_dma_ctrl_mem2dev (void)

void	dao_dma_compl_wait (uint16_t vchan)

static __rte_always_inline int	dao_dma_has_stats_feature (void)

static __rte_always_inline bool	dao_dma_op_status (struct dao_dma_vchan_state *vchan, uint16_t op_idx)

static __rte_always_inline bool	dao_dma_flush (struct dao_dma_vchan_state *vchan, const uint8_t avail)

static __rte_always_inline uint16_t	dao_dma_src_avail (struct dao_dma_vchan_state *vchan)

static __rte_always_inline uint16_t	dao_dma_dst_avail (struct dao_dma_vchan_state *vchan)

static __rte_always_inline struct rte_dma_sge *	dao_dma_sge_src (struct dao_dma_vchan_state *vchan)

static __rte_always_inline struct rte_dma_sge *	dao_dma_sge_dst (struct dao_dma_vchan_state *vchan)

static __rte_always_inline void	dao_dma_enq_x1 (struct dao_dma_vchan_state *vchan, rte_iova_t src, uint32_t src_len, rte_iova_t dst, uint32_t dst_len)

static __rte_always_inline void	dao_dma_enq_dst_x1 (struct dao_dma_vchan_state *vchan, rte_iova_t dst, uint32_t dst_len)

static __rte_always_inline void	dao_dma_enq_src_x1 (struct dao_dma_vchan_state *vchan, rte_iova_t src, uint32_t src_len)

static __rte_always_inline uint16_t	dao_dma_enq_x4 (struct dao_dma_vchan_state vchan, uint64x2_t vsrc, uint64x2_t *vdst)

static __rte_always_inline void	dao_dma_check_compl (struct dao_dma_vchan_state *vchan)

static __rte_always_inline void	dao_dma_check_meta_compl (struct dao_dma_vchan_state *vchan, const int mem_order)

static __rte_always_inline void	dao_dma_update_cmpl_meta (struct dao_dma_vchan_state vchan, uint16_t ptr, uint16_t val, uint16_t *pend_ptr, uint16_t pend_val, uint16_t tail)

static __rte_always_inline void	dao_dma_check_meta_compl_v2 (struct dao_dma_vchan_state *vchan, const int mem_order)

static __rte_always_inline void	dao_dma_update_cmpl_meta_v2 (struct dao_dma_vchan_state vchan, uint16_t ptr, uint16_t val, uint16_t tail)

Detailed Description

DAO DMA helper

Definition in file dao_dma.h.

Macro Definition Documentation

◆ DAO_DMA_MAX_POINTER

#define DAO_DMA_MAX_POINTER 15u

DMA MAX pointer

Definition at line 25 of file dao_dma.h.

◆ DAO_DMA_MAX_META_POINTER

#define DAO_DMA_MAX_META_POINTER 48

DMA MAX META pointer MAX DMA POINTER

Definition at line 28 of file dao_dma.h.

◆ DAO_DMA_MAX_POINTER_THR_DFLT

#define DAO_DMA_MAX_POINTER_THR_DFLT 8u

DMA pointer flush threshold

Definition at line 31 of file dao_dma.h.

◆ DAO_DMA_MAX_VCHAN_PER_LCORE

#define DAO_DMA_MAX_VCHAN_PER_LCORE 64

DMA Max VCHAN per lcore

Definition at line 34 of file dao_dma.h.

◆ DAO_DMA_MAX_INFLIGHT_MDATA

#define DAO_DMA_MAX_INFLIGHT_MDATA 4096

DMA inflight event meta data

Definition at line 37 of file dao_dma.h.

Function Documentation

◆ RTE_DECLARE_PER_LCORE()

RTE_DECLARE_PER_LCORE	(	struct dao_dma_vchan_info *	,
		dao_dma_vchan_info
	)

DMA per lcore vchan info

◆ dao_dma_flush_submit()

int dao_dma_flush_submit ( void )

Flush DMA requests and submit ops

Returns: Zero on success.

◆ dao_dma_stats_get()

int dao_dma_stats_get	(	uint16_t	lcore_id,
		struct dao_dma_stats *	stats
	)

Get DMA stats from DAO library

Parameters

lcore_id	Lcore to get stats from.
stats	Address to store stats.

Returns: Zero on success.

◆ dao_dma_lcore_dev2mem_set()

int dao_dma_lcore_dev2mem_set	(	int16_t	dma_devid,
		uint16_t	nb_vchans,
		uint16_t	flush_thr
	)

Assign dev2mem dma device to an lcore.

Parameters

dma_devid	DMA device id to assign to lcore.
nb_vchans	Number of vchans available in DMA device
flush_thr	Flush threshold.

Returns: Zero on success.

◆ dao_dma_lcore_mem2dev_set()

int dao_dma_lcore_mem2dev_set	(	int16_t	dma_devid,
		uint16_t	nb_vchans,
		uint16_t	flush_thr
	)

Assign mem2dev dma device to an lcore.

Parameters

dma_devid	DMA device id to assign to lcore.
nb_vchans	Number of vchans available in DMA device
flush_thr	Flush threshold.

Returns: Zero on success.

◆ dao_dma_lcore_mem2dev_autofree_set()

int dao_dma_lcore_mem2dev_autofree_set	(	int16_t	dma_devid,
		uint16_t	vchan,
		bool	enable
	)

Enable or Disable auto free on a mem2dev dma device's vchan of a given lcore.

Parameters

dma_devid	DMA device id to assigned to the lcore.
vchan	mem2dev vchan of the dma device.
enable	Flag

Returns: Zero on success.

◆ dao_dma_ctrl_dev_set()

int dao_dma_ctrl_dev_set	(	int16_t	dev2mem_id,
		int16_t	mem2dev_id
	)

Assign a global DMA device id for control path.

Parameters

dev2mem_id	dev2mem dma device id.
mem2dev_id	mem2dev dma device id.

Returns: Zero on success.

◆ dao_dma_ctrl_dev2mem()

int16_t dao_dma_ctrl_dev2mem ( void )

Get global DMA dev2mem device id.

Returns: dma device id.

◆ dao_dma_ctrl_mem2dev()

int16_t dao_dma_ctrl_mem2dev ( void )

Get global DMA mem2dev device id.

Returns: dma device id.

◆ dao_dma_compl_wait()

void dao_dma_compl_wait ( uint16_t vchan )

Check and wait for all DMA requests to complete

Parameters

vchan Vchan ID

◆ dao_dma_has_stats_feature()

static __rte_always_inline int dao_dma_has_stats_feature ( void )

static

Tests DMA stats support

Returns: 1 if DMA stats is supported, 0 otherwise.

Definition at line 237 of file dao_dma.h.

◆ dao_dma_op_status()

static __rte_always_inline bool dao_dma_op_status	(	struct dao_dma_vchan_state *	vchan,
		uint16_t	op_idx
	)

static

Get DMA operation status

Parameters

vchan	Vchan associated with DMA operation.
op_idx	DMA operation index

Returns: True - DMA op complete. False - DMA op not complete.

Definition at line 258 of file dao_dma.h.

◆ dao_dma_flush()

static __rte_always_inline bool dao_dma_flush	(	struct dao_dma_vchan_state *	vchan,
		const uint8_t	avail
	)

static

Flush DMA operation

Parameters

vchan	Vchan state pointer
avail	Avail needed post flush

Returns: True on success.

Definition at line 280 of file dao_dma.h.

◆ dao_dma_src_avail()

static __rte_always_inline uint16_t dao_dma_src_avail ( struct dao_dma_vchan_state * vchan )

static

Get available space in DMA src vchan state

Parameters

vchan Vchan state pointer

Returns: Returns space in number of src pointers.

Definition at line 319 of file dao_dma.h.

◆ dao_dma_dst_avail()

static __rte_always_inline uint16_t dao_dma_dst_avail ( struct dao_dma_vchan_state * vchan )

static

Get available space in DMA dst vchan state

Parameters

vchan Vchan state pointer

Returns: Returns space in number of dst pointers.

Definition at line 335 of file dao_dma.h.

◆ dao_dma_sge_src()

static __rte_always_inline struct rte_dma_sge * dao_dma_sge_src ( struct dao_dma_vchan_state * vchan )

static

Get DMA src pointer base

Parameters

vchan Vchan state pointer

Returns: Returns pointer base

Definition at line 351 of file dao_dma.h.

◆ dao_dma_sge_dst()

static __rte_always_inline struct rte_dma_sge * dao_dma_sge_dst ( struct dao_dma_vchan_state * vchan )

static

Get DMA dst pointer base

Parameters

vchan Vchan state pointer

Returns: Returns pointer base

Definition at line 365 of file dao_dma.h.

◆ dao_dma_enq_x1()

static __rte_always_inline void dao_dma_enq_x1	(	struct dao_dma_vchan_state *	vchan,
		rte_iova_t	src,
		uint32_t	src_len,
		rte_iova_t	dst,
		uint32_t	dst_len
	)

static

Enqueue one DMA pointer pair.

Space is vchan state is checked by caller before calling this API

Parameters

vchan	Vchan state pointer
src	source data IOVA
src_len	source data length
dst	Destination data IOVA
dst_len	Destination data len

Definition at line 388 of file dao_dma.h.

◆ dao_dma_enq_dst_x1()

static __rte_always_inline void dao_dma_enq_dst_x1	(	struct dao_dma_vchan_state *	vchan,
		rte_iova_t	dst,
		uint32_t	dst_len
	)

static

Enqueue one DMA pointer for destination address.

Space in vchan state is checked by caller before calling this API

Parameters

vchan	Vchan state pointer
dst	Destination data IOVA
dst_len	Destination data len

Definition at line 417 of file dao_dma.h.

◆ dao_dma_enq_src_x1()

static __rte_always_inline void dao_dma_enq_src_x1	(	struct dao_dma_vchan_state *	vchan,
		rte_iova_t	src,
		uint32_t	src_len
	)

static

Enqueue one DMA pointer for source address.

Space in vchan state is checked by caller before calling this API

Parameters

vchan	Vchan state pointer
src	source data IOVA
src_len	source data length

Definition at line 441 of file dao_dma.h.

◆ dao_dma_enq_x4()

static __rte_always_inline uint16_t dao_dma_enq_x4	(	struct dao_dma_vchan_state *	vchan,
		uint64x2_t *	vsrc,
		uint64x2_t *	vdst
	)

static

Enqueue four DMA pointer pairs.

Parameters

vchan	Vchan state pointer
vsrc	source vector in format of `struct rte_dma_sge`.
vdst	destination vector in format of `struct rte_dma_sge`.

Returns: Number of pointers enqueued.

Definition at line 464 of file dao_dma.h.

◆ dao_dma_check_compl()

static __rte_always_inline void dao_dma_check_compl ( struct dao_dma_vchan_state * vchan )

static

Check and update DMA completions.

Parameters

vchan Vchan state pointer

Definition at line 531 of file dao_dma.h.

◆ dao_dma_check_meta_compl()

static __rte_always_inline void dao_dma_check_meta_compl	(	struct dao_dma_vchan_state *	vchan,
		const int	mem_order
	)

static

Check and update DMA completions with metadata.

Parameters

vchan	Vchan state pointer
mem_order	Memory order to update address

Definition at line 555 of file dao_dma.h.

◆ dao_dma_update_cmpl_meta()

static __rte_always_inline void dao_dma_update_cmpl_meta	(	struct dao_dma_vchan_state *	vchan,
		uint16_t *	ptr,
		uint16_t	val,
		uint16_t *	pend_ptr,
		uint16_t	pend_val,
		uint16_t	tail
	)

static

Update DMA op index metadata.

Parameters

vchan	Vchan state pointer
ptr	desc location to write
val	Value to store in descriptor location
pend_ptr	Pending count location to write
pend_val	Pending Value to store
tail	Meta data index to store

Definition at line 598 of file dao_dma.h.

◆ dao_dma_check_meta_compl_v2()

static __rte_always_inline void dao_dma_check_meta_compl_v2	(	struct dao_dma_vchan_state *	vchan,
		const int	mem_order
	)

static

Check and update DMA completions with metadata version2.

Parameters

vchan	Vchan state pointer
mem_order	Memory order to update address

Definition at line 621 of file dao_dma.h.

◆ dao_dma_update_cmpl_meta_v2()

static __rte_always_inline void dao_dma_update_cmpl_meta_v2	(	struct dao_dma_vchan_state *	vchan,
		uint16_t *	ptr,
		uint16_t	val,
		uint16_t	tail
	)

static

Update DMA op index metadata version2.

Parameters

vchan	Vchan state pointer
ptr	desc location to write
val	Value to store in descriptor location
tail	Meta data index to store

Definition at line 659 of file dao_dma.h.

Data Structures

Macros

Functions

Detailed Description

Macro Definition Documentation

◆ DAO_DMA_MAX_POINTER

◆ DAO_DMA_MAX_META_POINTER

◆ DAO_DMA_MAX_POINTER_THR_DFLT

◆ DAO_DMA_MAX_VCHAN_PER_LCORE

◆ DAO_DMA_MAX_INFLIGHT_MDATA

Function Documentation

◆ RTE_DECLARE_PER_LCORE()

◆ dao_dma_flush_submit()

◆ dao_dma_stats_get()

◆ dao_dma_lcore_dev2mem_set()

◆ dao_dma_lcore_mem2dev_set()

◆ dao_dma_lcore_mem2dev_autofree_set()

◆ dao_dma_ctrl_dev_set()

◆ dao_dma_ctrl_dev2mem()

◆ dao_dma_ctrl_mem2dev()

◆ dao_dma_compl_wait()

◆ dao_dma_has_stats_feature()

◆ dao_dma_op_status()

◆ dao_dma_flush()

◆ dao_dma_src_avail()

◆ dao_dma_dst_avail()

◆ dao_dma_sge_src()

◆ dao_dma_sge_dst()

◆ dao_dma_enq_x1()

◆ dao_dma_enq_dst_x1()

◆ dao_dma_enq_src_x1()

◆ dao_dma_enq_x4()

◆ dao_dma_check_compl()

◆ dao_dma_check_meta_compl()

◆ dao_dma_update_cmpl_meta()

◆ dao_dma_check_meta_compl_v2()

◆ dao_dma_update_cmpl_meta_v2()